2024-11-21 21:23:42,916 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 21:23:42,930 main DEBUG Took 0.012161 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 21:23:42,931 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 21:23:42,931 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 21:23:42,933 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 21:23:42,934 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,944 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 21:23:42,958 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,961 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,963 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,965 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,966 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,966 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,968 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,969 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,970 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,971 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,972 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,973 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,975 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:23:42,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,976 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 21:23:42,978 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:23:42,980 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 21:23:42,982 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 21:23:42,983 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 21:23:42,985 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 21:23:42,985 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 21:23:42,996 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 21:23:43,000 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 21:23:43,003 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 21:23:43,003 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 21:23:43,004 main DEBUG createAppenders(={Console}) 2024-11-21 21:23:43,005 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-21 21:23:43,005 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 21:23:43,006 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-21 21:23:43,006 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 21:23:43,007 main DEBUG OutputStream closed 2024-11-21 21:23:43,007 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 21:23:43,007 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 21:23:43,008 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-21 21:23:43,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 21:23:43,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 21:23:43,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 21:23:43,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 21:23:43,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 21:23:43,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 21:23:43,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 21:23:43,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 21:23:43,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 21:23:43,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 21:23:43,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 21:23:43,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 21:23:43,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 21:23:43,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 21:23:43,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 21:23:43,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 21:23:43,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 21:23:43,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 21:23:43,122 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 21:23:43,122 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-21 21:23:43,122 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 21:23:43,124 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-21T21:23:43,414 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f 2024-11-21 21:23:43,418 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 21:23:43,419 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T21:23:43,431 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-21T21:23:43,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T21:23:43,480 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662, deleteOnExit=true 2024-11-21T21:23:43,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T21:23:43,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/test.cache.data in system properties and HBase conf 2024-11-21T21:23:43,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T21:23:43,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/hadoop.log.dir in system properties and HBase conf 2024-11-21T21:23:43,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T21:23:43,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T21:23:43,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T21:23:43,587 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T21:23:43,687 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:23:43,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T21:23:43,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T21:23:43,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T21:23:43,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T21:23:43,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T21:23:43,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T21:23:43,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T21:23:43,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T21:23:43,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T21:23:43,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/nfs.dump.dir in system properties and HBase conf 2024-11-21T21:23:43,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/java.io.tmpdir in system properties and HBase conf 2024-11-21T21:23:43,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T21:23:43,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T21:23:43,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T21:23:44,879 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T21:23:44,952 INFO [Time-limited test {}] log.Log(170): Logging initialized @3072ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T21:23:45,021 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T21:23:45,080 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T21:23:45,100 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T21:23:45,100 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T21:23:45,101 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T21:23:45,112 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T21:23:45,115 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/hadoop.log.dir/,AVAILABLE} 2024-11-21T21:23:45,116 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T21:23:45,305 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/java.io.tmpdir/jetty-localhost-41405-hadoop-hdfs-3_4_1-tests_jar-_-any-3642531779453917517/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T21:23:45,312 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41405} 2024-11-21T21:23:45,312 INFO [Time-limited test {}] server.Server(415): Started @3433ms 2024-11-21T21:23:45,843 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T21:23:45,853 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T21:23:45,854 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T21:23:45,854 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T21:23:45,855 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T21:23:45,855 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/hadoop.log.dir/,AVAILABLE} 2024-11-21T21:23:45,856 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T21:23:45,967 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/java.io.tmpdir/jetty-localhost-42427-hadoop-hdfs-3_4_1-tests_jar-_-any-9979831192152889390/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T21:23:45,967 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42427} 2024-11-21T21:23:45,968 INFO [Time-limited test {}] server.Server(415): Started @4088ms 2024-11-21T21:23:46,019 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T21:23:47,053 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/data/data1/current/BP-1334735833-172.17.0.2-1732224224328/current, will proceed with Du for space computation calculation, 2024-11-21T21:23:47,053 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/data/data2/current/BP-1334735833-172.17.0.2-1732224224328/current, will proceed with Du for space computation calculation, 2024-11-21T21:23:47,079 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T21:23:47,124 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2244a22d19c17965 with lease ID 0x52b398801ce52d23: Processing first storage report for DS-5519525f-4d6d-4c22-ab3c-a305a0d48fb9 from datanode DatanodeRegistration(127.0.0.1:33455, datanodeUuid=c7e979b6-f1e6-4ae2-a37e-979b86f3c671, infoPort=34653, infoSecurePort=0, ipcPort=42901, storageInfo=lv=-57;cid=testClusterID;nsid=504620972;c=1732224224328) 2024-11-21T21:23:47,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2244a22d19c17965 with lease ID 0x52b398801ce52d23: from storage DS-5519525f-4d6d-4c22-ab3c-a305a0d48fb9 node DatanodeRegistration(127.0.0.1:33455, datanodeUuid=c7e979b6-f1e6-4ae2-a37e-979b86f3c671, infoPort=34653, infoSecurePort=0, ipcPort=42901, storageInfo=lv=-57;cid=testClusterID;nsid=504620972;c=1732224224328), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T21:23:47,126 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2244a22d19c17965 with lease ID 0x52b398801ce52d23: Processing first storage report for DS-0466723f-2cfe-4ca1-82b0-2f869f9afd29 from datanode DatanodeRegistration(127.0.0.1:33455, datanodeUuid=c7e979b6-f1e6-4ae2-a37e-979b86f3c671, infoPort=34653, infoSecurePort=0, ipcPort=42901, storageInfo=lv=-57;cid=testClusterID;nsid=504620972;c=1732224224328) 2024-11-21T21:23:47,127 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2244a22d19c17965 with lease ID 0x52b398801ce52d23: from storage DS-0466723f-2cfe-4ca1-82b0-2f869f9afd29 node DatanodeRegistration(127.0.0.1:33455, datanodeUuid=c7e979b6-f1e6-4ae2-a37e-979b86f3c671, infoPort=34653, infoSecurePort=0, ipcPort=42901, storageInfo=lv=-57;cid=testClusterID;nsid=504620972;c=1732224224328), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T21:23:47,142 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f 2024-11-21T21:23:47,204 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/zookeeper_0, clientPort=63066, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T21:23:47,213 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63066 2024-11-21T21:23:47,223 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:23:47,225 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:23:47,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741825_1001 (size=7) 2024-11-21T21:23:47,842 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b with version=8 2024-11-21T21:23:47,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/hbase-staging 2024-11-21T21:23:47,938 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T21:23:48,148 INFO [Time-limited test {}] client.ConnectionUtils(128): master/849dee24a242:0 server-side Connection retries=45 2024-11-21T21:23:48,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:23:48,158 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:23:48,163 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:23:48,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:23:48,164 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:23:48,320 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T21:23:48,373 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T21:23:48,382 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T21:23:48,385 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:23:48,407 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32815 (auto-detected) 2024-11-21T21:23:48,408 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T21:23:48,425 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35047 2024-11-21T21:23:48,446 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35047 connecting to ZooKeeper ensemble=127.0.0.1:63066 2024-11-21T21:23:48,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:350470x0, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:23:48,562 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35047-0x1015f4813200000 connected 2024-11-21T21:23:48,962 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:23:48,965 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:23:48,972 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:23:48,975 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b, hbase.cluster.distributed=false 2024-11-21T21:23:48,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:23:49,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35047 2024-11-21T21:23:49,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35047 2024-11-21T21:23:49,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35047 2024-11-21T21:23:49,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35047 2024-11-21T21:23:49,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35047 2024-11-21T21:23:49,122 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/849dee24a242:0 server-side Connection retries=45 2024-11-21T21:23:49,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:23:49,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:23:49,125 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:23:49,125 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:23:49,125 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:23:49,128 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T21:23:49,130 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:23:49,131 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34123 2024-11-21T21:23:49,134 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34123 connecting to ZooKeeper ensemble=127.0.0.1:63066 2024-11-21T21:23:49,135 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:23:49,138 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:23:49,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341230x0, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:23:49,154 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:341230x0, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:23:49,155 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34123-0x1015f4813200001 connected 2024-11-21T21:23:49,158 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T21:23:49,167 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T21:23:49,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T21:23:49,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:23:49,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34123 2024-11-21T21:23:49,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34123 2024-11-21T21:23:49,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34123 2024-11-21T21:23:49,185 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34123 2024-11-21T21:23:49,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34123 2024-11-21T21:23:49,205 DEBUG [M:0;849dee24a242:35047 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;849dee24a242:35047 2024-11-21T21:23:49,208 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/849dee24a242,35047,1732224227985 2024-11-21T21:23:49,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:23:49,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:23:49,300 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/849dee24a242,35047,1732224227985 2024-11-21T21:23:49,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:49,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T21:23:49,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:49,350 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T21:23:49,351 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/849dee24a242,35047,1732224227985 from backup master directory 2024-11-21T21:23:49,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/849dee24a242,35047,1732224227985 2024-11-21T21:23:49,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:23:49,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:23:49,365 WARN [master/849dee24a242:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:23:49,365 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=849dee24a242,35047,1732224227985 2024-11-21T21:23:49,367 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T21:23:49,368 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T21:23:49,439 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/hbase.id] with ID: 38eb55c4-5488-4e84-a66c-60c020ae062e 2024-11-21T21:23:49,439 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/.tmp/hbase.id 2024-11-21T21:23:49,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741826_1002 (size=42) 2024-11-21T21:23:49,857 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/.tmp/hbase.id]:[hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/hbase.id] 2024-11-21T21:23:49,911 INFO [master/849dee24a242:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:23:49,917 INFO [master/849dee24a242:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T21:23:49,938 INFO [master/849dee24a242:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-21T21:23:49,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:49,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:50,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741827_1003 (size=196) 2024-11-21T21:23:50,433 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:23:50,436 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T21:23:50,456 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:50,461 INFO [master/849dee24a242:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:23:50,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741828_1004 (size=1189) 2024-11-21T21:23:50,923 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store 2024-11-21T21:23:50,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741829_1005 (size=34) 2024-11-21T21:23:51,347 INFO [master/849dee24a242:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T21:23:51,351 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:23:51,352 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:23:51,352 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:23:51,352 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:23:51,354 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:23:51,354 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:23:51,354 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:23:51,355 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732224231352Disabling compacts and flushes for region at 1732224231352Disabling writes for close at 1732224231354 (+2 ms)Writing region close event to WAL at 1732224231354Closed at 1732224231354 2024-11-21T21:23:51,357 WARN [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/.initializing 2024-11-21T21:23:51,358 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/WALs/849dee24a242,35047,1732224227985 2024-11-21T21:23:51,366 INFO [master/849dee24a242:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:23:51,382 INFO [master/849dee24a242:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849dee24a242%2C35047%2C1732224227985, suffix=, logDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/WALs/849dee24a242,35047,1732224227985, archiveDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/oldWALs, maxLogs=10 2024-11-21T21:23:51,405 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/WALs/849dee24a242,35047,1732224227985/849dee24a242%2C35047%2C1732224227985.1732224231387, exclude list is [], retry=0 2024-11-21T21:23:51,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33455,DS-5519525f-4d6d-4c22-ab3c-a305a0d48fb9,DISK] 2024-11-21T21:23:51,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T21:23:51,461 INFO [master/849dee24a242:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/WALs/849dee24a242,35047,1732224227985/849dee24a242%2C35047%2C1732224227985.1732224231387 2024-11-21T21:23:51,462 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34653:34653)] 2024-11-21T21:23:51,462 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:23:51,463 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:23:51,466 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,467 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,538 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T21:23:51,543 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:51,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:51,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T21:23:51,551 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:51,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:23:51,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T21:23:51,557 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:51,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:23:51,559 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T21:23:51,562 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:51,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:23:51,564 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,568 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,570 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,575 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,576 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,580 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:23:51,585 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:23:51,590 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:23:51,592 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69984150, jitterRate=0.04284510016441345}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:23:51,600 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732224231479Initializing all the Stores at 1732224231482 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732224231482Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224231483 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224231483Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224231484 (+1 ms)Cleaning up temporary data from old regions at 1732224231576 (+92 ms)Region opened successfully at 1732224231600 (+24 ms) 2024-11-21T21:23:51,602 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T21:23:51,640 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464d2f4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=849dee24a242/172.17.0.2:0 2024-11-21T21:23:51,671 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T21:23:51,682 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T21:23:51,682 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T21:23:51,685 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T21:23:51,686 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T21:23:51,692 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T21:23:51,692 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T21:23:51,718 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T21:23:51,726 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T21:23:51,781 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T21:23:51,783 INFO [master/849dee24a242:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T21:23:51,787 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T21:23:51,795 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T21:23:51,797 INFO [master/849dee24a242:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T21:23:51,801 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T21:23:51,812 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T21:23:51,814 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T21:23:51,827 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T21:23:51,849 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T21:23:51,858 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T21:23:51,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:23:51,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:23:51,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:51,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:51,871 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=849dee24a242,35047,1732224227985, sessionid=0x1015f4813200000, setting cluster-up flag (Was=false) 2024-11-21T21:23:51,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:51,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:52,069 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T21:23:52,071 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=849dee24a242,35047,1732224227985 2024-11-21T21:23:52,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:52,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:52,226 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T21:23:52,229 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=849dee24a242,35047,1732224227985 2024-11-21T21:23:52,236 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T21:23:52,295 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(746): ClusterId : 38eb55c4-5488-4e84-a66c-60c020ae062e 2024-11-21T21:23:52,298 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T21:23:52,302 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T21:23:52,311 INFO [master/849dee24a242:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T21:23:52,313 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T21:23:52,313 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T21:23:52,317 INFO [master/849dee24a242:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T21:23:52,323 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T21:23:52,324 DEBUG [RS:0;849dee24a242:34123 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126e8da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=849dee24a242/172.17.0.2:0 2024-11-21T21:23:52,324 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 849dee24a242,35047,1732224227985 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T21:23:52,333 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:23:52,334 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:23:52,334 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:23:52,334 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:23:52,334 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/849dee24a242:0, corePoolSize=10, maxPoolSize=10 2024-11-21T21:23:52,335 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,335 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/849dee24a242:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:23:52,335 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,342 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732224262342 2024-11-21T21:23:52,344 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;849dee24a242:34123 2024-11-21T21:23:52,344 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T21:23:52,345 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:23:52,345 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T21:23:52,346 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T21:23:52,349 INFO [RS:0;849dee24a242:34123 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T21:23:52,349 INFO [RS:0;849dee24a242:34123 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T21:23:52,349 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T21:23:52,350 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T21:23:52,351 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T21:23:52,351 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T21:23:52,352 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T21:23:52,352 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(2659): reportForDuty to master=849dee24a242,35047,1732224227985 with port=34123, startcode=1732224229084 2024-11-21T21:23:52,353 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:52,352 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,354 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T21:23:52,357 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T21:23:52,359 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T21:23:52,360 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T21:23:52,364 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T21:23:52,365 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T21:23:52,366 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.large.0-1732224232366,5,FailOnTimeoutGroup] 2024-11-21T21:23:52,367 DEBUG [RS:0;849dee24a242:34123 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T21:23:52,368 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.small.0-1732224232366,5,FailOnTimeoutGroup] 2024-11-21T21:23:52,368 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,368 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T21:23:52,370 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,370 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741831_1007 (size=1321) 2024-11-21T21:23:52,379 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T21:23:52,380 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:23:52,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741832_1008 (size=32) 2024-11-21T21:23:52,410 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51863, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T21:23:52,417 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35047 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 849dee24a242,34123,1732224229084 2024-11-21T21:23:52,421 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35047 {}] master.ServerManager(517): Registering regionserver=849dee24a242,34123,1732224229084 2024-11-21T21:23:52,440 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:23:52,441 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33491 2024-11-21T21:23:52,441 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T21:23:52,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:23:52,455 DEBUG [RS:0;849dee24a242:34123 {}] zookeeper.ZKUtil(111): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/849dee24a242,34123,1732224229084 2024-11-21T21:23:52,455 WARN [RS:0;849dee24a242:34123 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:23:52,456 INFO [RS:0;849dee24a242:34123 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:23:52,456 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/WALs/849dee24a242,34123,1732224229084 2024-11-21T21:23:52,458 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [849dee24a242,34123,1732224229084] 2024-11-21T21:23:52,486 INFO [RS:0;849dee24a242:34123 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T21:23:52,504 INFO [RS:0;849dee24a242:34123 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T21:23:52,509 INFO [RS:0;849dee24a242:34123 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T21:23:52,509 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,510 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T21:23:52,516 INFO [RS:0;849dee24a242:34123 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T21:23:52,517 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,518 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,518 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,518 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,518 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,518 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,519 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/849dee24a242:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:23:52,519 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,519 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,519 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,519 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,519 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,520 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:23:52,520 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/849dee24a242:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:23:52,520 DEBUG [RS:0;849dee24a242:34123 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:23:52,521 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,521 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,521 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,522 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,522 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,522 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,34123,1732224229084-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:23:52,546 INFO [RS:0;849dee24a242:34123 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T21:23:52,548 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,34123,1732224229084-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,549 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,549 INFO [RS:0;849dee24a242:34123 {}] regionserver.Replication(171): 849dee24a242,34123,1732224229084 started 2024-11-21T21:23:52,566 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:52,567 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1482): Serving as 849dee24a242,34123,1732224229084, RpcServer on 849dee24a242/172.17.0.2:34123, sessionid=0x1015f4813200001 2024-11-21T21:23:52,567 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T21:23:52,568 DEBUG [RS:0;849dee24a242:34123 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 849dee24a242,34123,1732224229084 2024-11-21T21:23:52,568 DEBUG [RS:0;849dee24a242:34123 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '849dee24a242,34123,1732224229084' 2024-11-21T21:23:52,568 DEBUG [RS:0;849dee24a242:34123 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T21:23:52,569 DEBUG [RS:0;849dee24a242:34123 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T21:23:52,570 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T21:23:52,570 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T21:23:52,570 DEBUG [RS:0;849dee24a242:34123 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 849dee24a242,34123,1732224229084 2024-11-21T21:23:52,570 DEBUG [RS:0;849dee24a242:34123 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '849dee24a242,34123,1732224229084' 2024-11-21T21:23:52,570 DEBUG [RS:0;849dee24a242:34123 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T21:23:52,571 DEBUG [RS:0;849dee24a242:34123 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T21:23:52,571 DEBUG [RS:0;849dee24a242:34123 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T21:23:52,571 INFO [RS:0;849dee24a242:34123 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T21:23:52,572 INFO [RS:0;849dee24a242:34123 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T21:23:52,676 INFO [RS:0;849dee24a242:34123 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:23:52,680 INFO [RS:0;849dee24a242:34123 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849dee24a242%2C34123%2C1732224229084, suffix=, logDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/WALs/849dee24a242,34123,1732224229084, archiveDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/oldWALs, maxLogs=32 2024-11-21T21:23:52,695 DEBUG [RS:0;849dee24a242:34123 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/WALs/849dee24a242,34123,1732224229084/849dee24a242%2C34123%2C1732224229084.1732224232682, exclude list is [], retry=0 2024-11-21T21:23:52,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33455,DS-5519525f-4d6d-4c22-ab3c-a305a0d48fb9,DISK] 2024-11-21T21:23:52,704 INFO [RS:0;849dee24a242:34123 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/WALs/849dee24a242,34123,1732224229084/849dee24a242%2C34123%2C1732224229084.1732224232682 2024-11-21T21:23:52,704 DEBUG [RS:0;849dee24a242:34123 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34653:34653)] 2024-11-21T21:23:52,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:23:52,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T21:23:52,823 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:23:52,824 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:52,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:52,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T21:23:52,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:23:52,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:52,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:52,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T21:23:52,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:23:52,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:52,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:52,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T21:23:52,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:23:52,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:52,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:52,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:23:52,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740 2024-11-21T21:23:52,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740 2024-11-21T21:23:52,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:23:52,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:23:52,849 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:23:52,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:23:52,856 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:23:52,857 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67493374, jitterRate=0.005729645490646362}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:23:52,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732224232809Initializing all the Stores at 1732224232811 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732224232811Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732224232811Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224232812 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732224232812Cleaning up temporary data from old regions at 1732224232848 (+36 ms)Region opened successfully at 1732224232859 (+11 ms) 2024-11-21T21:23:52,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:23:52,860 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:23:52,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:23:52,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:23:52,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:23:52,861 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:23:52,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732224232859Disabling compacts and flushes for region at 1732224232859Disabling writes for close at 1732224232860 (+1 ms)Writing region close event to WAL at 1732224232861 (+1 ms)Closed at 1732224232861 2024-11-21T21:23:52,866 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:23:52,866 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T21:23:52,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T21:23:52,885 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:23:52,889 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T21:23:53,042 DEBUG [849dee24a242:35047 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T21:23:53,052 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:23:53,061 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 849dee24a242,34123,1732224229084, state=OPENING 2024-11-21T21:23:53,114 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T21:23:53,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:53,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:23:53,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:23:53,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:23:53,199 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:23:53,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:23:53,379 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T21:23:53,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46097, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T21:23:53,394 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T21:23:53,395 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:23:53,396 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T21:23:53,399 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849dee24a242%2C34123%2C1732224229084.meta, suffix=.meta, logDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/WALs/849dee24a242,34123,1732224229084, archiveDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/oldWALs, maxLogs=32 2024-11-21T21:23:53,415 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/WALs/849dee24a242,34123,1732224229084/849dee24a242%2C34123%2C1732224229084.meta.1732224233401.meta, exclude list is [], retry=0 2024-11-21T21:23:53,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33455,DS-5519525f-4d6d-4c22-ab3c-a305a0d48fb9,DISK] 2024-11-21T21:23:53,422 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/WALs/849dee24a242,34123,1732224229084/849dee24a242%2C34123%2C1732224229084.meta.1732224233401.meta 2024-11-21T21:23:53,423 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34653:34653)] 2024-11-21T21:23:53,423 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:23:53,425 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T21:23:53,427 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T21:23:53,431 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T21:23:53,435 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T21:23:53,435 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:23:53,435 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T21:23:53,435 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T21:23:53,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T21:23:53,440 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:23:53,440 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:53,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:53,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T21:23:53,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:23:53,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:53,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:53,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T21:23:53,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:23:53,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:53,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:53,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T21:23:53,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:23:53,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:53,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:23:53,449 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:23:53,450 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740 2024-11-21T21:23:53,453 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740 2024-11-21T21:23:53,455 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:23:53,455 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:23:53,456 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:23:53,459 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:23:53,460 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59182900, jitterRate=-0.11810606718063354}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:23:53,461 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T21:23:53,462 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732224233436Writing region info on filesystem at 1732224233436Initializing all the Stores at 1732224233438 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732224233438Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732224233438Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224233438Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732224233438Cleaning up temporary data from old regions at 1732224233455 (+17 ms)Running coprocessor post-open hooks at 1732224233461 (+6 ms)Region opened successfully at 1732224233462 (+1 ms) 2024-11-21T21:23:53,469 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732224233371 2024-11-21T21:23:53,478 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T21:23:53,478 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T21:23:53,480 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:23:53,484 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 849dee24a242,34123,1732224229084, state=OPEN 2024-11-21T21:23:53,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:23:53,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:23:53,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:23:53,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:23:53,581 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:53,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T21:23:53,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=849dee24a242,34123,1732224229084 in 380 msec 2024-11-21T21:23:53,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T21:23:53,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 718 msec 2024-11-21T21:23:53,597 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:23:53,597 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T21:23:53,615 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:53,616 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:53,633 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:53,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55819, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:53,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3920 sec 2024-11-21T21:23:53,656 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732224233656, completionTime=-1 2024-11-21T21:23:53,659 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T21:23:53,659 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T21:23:53,684 INFO [master/849dee24a242:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-21T21:23:53,684 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732224293684 2024-11-21T21:23:53,684 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732224353684 2024-11-21T21:23:53,684 INFO [master/849dee24a242:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-21T21:23:53,686 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,35047,1732224227985-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:53,686 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,35047,1732224227985-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:53,686 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,35047,1732224227985-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:53,688 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-849dee24a242:35047, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:53,688 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:53,689 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:53,694 DEBUG [master/849dee24a242:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T21:23:53,730 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.365sec 2024-11-21T21:23:53,731 INFO [master/849dee24a242:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T21:23:53,733 INFO [master/849dee24a242:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T21:23:53,733 INFO [master/849dee24a242:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T21:23:53,734 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T21:23:53,734 INFO [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T21:23:53,737 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,35047,1732224227985-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:23:53,738 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,35047,1732224227985-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T21:23:53,745 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T21:23:53,746 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T21:23:53,747 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,35047,1732224227985-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:23:53,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:53,809 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T21:23:53,809 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T21:23:53,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:53,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:53,822 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@386b99cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:53,833 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:53,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:53,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:53,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:53,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=849dee24a242,35047,1732224227985 2024-11-21T21:23:53,877 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=526, ProcessCount=11, AvailableMemoryMB=12946 2024-11-21T21:23:53,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:23:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,35047,1732224227985 2024-11-21T21:23:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79901794 2024-11-21T21:23:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:23:53,900 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:23:53,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:23:53,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:23:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:23:53,923 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:23:53,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T21:23:53,927 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:53,930 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:23:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:23:53,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741835_1011 (size=959) 2024-11-21T21:23:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:23:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:23:54,348 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:23:54,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741836_1012 (size=53) 2024-11-21T21:23:54,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:23:54,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4b5690ae4b378a5d396cbe15e83372d1, disabling compactions & flushes 2024-11-21T21:23:54,358 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:54,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:54,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. after waiting 0 ms 2024-11-21T21:23:54,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:54,358 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:54,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4b5690ae4b378a5d396cbe15e83372d1: Waiting for close lock at 1732224234358Disabling compacts and flushes for region at 1732224234358Disabling writes for close at 1732224234358Writing region close event to WAL at 1732224234358Closed at 1732224234358 2024-11-21T21:23:54,361 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:23:54,367 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732224234361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732224234361"}]},"ts":"1732224234361"} 2024-11-21T21:23:54,373 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:23:54,375 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:23:54,379 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224234375"}]},"ts":"1732224234375"} 2024-11-21T21:23:54,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:23:54,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, ASSIGN}] 2024-11-21T21:23:54,390 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, ASSIGN 2024-11-21T21:23:54,393 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, ASSIGN; state=OFFLINE, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=false 2024-11-21T21:23:54,544 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4b5690ae4b378a5d396cbe15e83372d1, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:23:54,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, ASSIGN because future has completed 2024-11-21T21:23:54,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:23:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:23:54,713 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:54,713 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:23:54,714 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,714 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:23:54,714 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,714 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,717 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,720 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5690ae4b378a5d396cbe15e83372d1 columnFamilyName A 2024-11-21T21:23:54,720 DEBUG [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:54,721 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] regionserver.HStore(327): Store=4b5690ae4b378a5d396cbe15e83372d1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:23:54,721 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,724 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5690ae4b378a5d396cbe15e83372d1 columnFamilyName B 2024-11-21T21:23:54,724 DEBUG [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:54,725 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] regionserver.HStore(327): Store=4b5690ae4b378a5d396cbe15e83372d1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:23:54,725 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,727 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5690ae4b378a5d396cbe15e83372d1 columnFamilyName C 2024-11-21T21:23:54,727 DEBUG [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:23:54,728 INFO [StoreOpener-4b5690ae4b378a5d396cbe15e83372d1-1 {}] regionserver.HStore(327): Store=4b5690ae4b378a5d396cbe15e83372d1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:23:54,728 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,729 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,730 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,732 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,732 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,733 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:23:54,735 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,738 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:23:54,739 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4b5690ae4b378a5d396cbe15e83372d1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64260774, jitterRate=-0.04243984818458557}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:23:54,739 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:54,740 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4b5690ae4b378a5d396cbe15e83372d1: Running coprocessor pre-open hook at 1732224234714Writing region info on filesystem at 1732224234714Initializing all the Stores at 1732224234716 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224234716Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224234717 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224234717Cleaning up temporary data from old regions at 1732224234732 (+15 ms)Running coprocessor post-open hooks at 1732224234739 (+7 ms)Region opened successfully at 1732224234740 (+1 ms) 2024-11-21T21:23:54,742 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., pid=6, masterSystemTime=1732224234705 2024-11-21T21:23:54,746 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:54,746 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:54,747 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4b5690ae4b378a5d396cbe15e83372d1, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:23:54,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:23:54,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T21:23:54,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 in 203 msec 2024-11-21T21:23:54,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T21:23:54,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, ASSIGN in 370 msec 2024-11-21T21:23:54,764 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:23:54,764 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224234764"}]},"ts":"1732224234764"} 2024-11-21T21:23:54,767 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:23:54,769 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:23:54,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 857 msec 2024-11-21T21:23:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:23:55,071 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:23:55,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,077 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,078 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bff7409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,083 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2538dc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,086 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,114 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:23:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,35047,1732224227985 2024-11-21T21:23:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@481dbf66 2024-11-21T21:23:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:23:55,122 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:23:55,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,133 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,141 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,166 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,175 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b3fb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106f4532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,193 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccd238a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,198 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6824dd94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,201 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c0a0fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,205 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b8551d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517e68dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,214 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1b5617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,217 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425be669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:23:55,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:23:55,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:23:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:23:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:23:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373a2bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:23:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:23:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:23:55,224 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:23:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33abc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:23:55,231 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,231 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,231 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,231 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,232 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,232 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,233 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,237 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,239 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,241 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,245 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,246 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:23:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:23:55,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:23:55,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:23:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:23:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:23:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T21:23:55,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:23:55,275 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:23:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:23:55,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:23:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:23:55,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:23:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:23:55,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:23:55,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:55,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:55,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:55,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/8748bf12e7fe4333bd88aedc7b5808b7 is 50, key is test_row_0/A:col10/1732224235292/Put/seqid=0 2024-11-21T21:23:55,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741837_1013 (size=16681) 2024-11-21T21:23:55,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/8748bf12e7fe4333bd88aedc7b5808b7 2024-11-21T21:23:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:23:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224295576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224295578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224295579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224295592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224295583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:23:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:55,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2480f351344f47aaa997080548b24ba6 is 50, key is test_row_0/B:col10/1732224235292/Put/seqid=0 2024-11-21T21:23:55,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224295752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224295751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224295753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224295753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224295754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741838_1014 (size=12001) 2024-11-21T21:23:55,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:23:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:55,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:23:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224295960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224295972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224295973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224295973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:55,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224295973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:23:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2480f351344f47aaa997080548b24ba6 2024-11-21T21:23:56,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:23:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:56,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/8662333747ae497bb8c8ef3b76d3e54e is 50, key is test_row_0/C:col10/1732224235292/Put/seqid=0 2024-11-21T21:23:56,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741839_1015 (size=12001) 2024-11-21T21:23:56,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/8662333747ae497bb8c8ef3b76d3e54e 2024-11-21T21:23:56,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224296270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/8748bf12e7fe4333bd88aedc7b5808b7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/8748bf12e7fe4333bd88aedc7b5808b7 2024-11-21T21:23:56,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224296296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224296298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/8748bf12e7fe4333bd88aedc7b5808b7, entries=250, sequenceid=14, filesize=16.3 K 2024-11-21T21:23:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224296298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T21:23:56,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224296303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2480f351344f47aaa997080548b24ba6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2480f351344f47aaa997080548b24ba6 2024-11-21T21:23:56,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:23:56,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:56,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2480f351344f47aaa997080548b24ba6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:23:56,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/8662333747ae497bb8c8ef3b76d3e54e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8662333747ae497bb8c8ef3b76d3e54e 2024-11-21T21:23:56,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8662333747ae497bb8c8ef3b76d3e54e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:23:56,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4b5690ae4b378a5d396cbe15e83372d1 in 1039ms, sequenceid=14, compaction requested=false 2024-11-21T21:23:56,384 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T21:23:56,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:23:56,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:23:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,494 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:23:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0f3e41bd59154355af4fd1666ada9c81 is 50, key is test_row_0/A:col10/1732224235510/Put/seqid=0 2024-11-21T21:23:56,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741840_1016 (size=12001) 2024-11-21T21:23:56,540 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0f3e41bd59154355af4fd1666ada9c81 2024-11-21T21:23:56,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b96b12779101462fbda0b3dddeb79d8e is 50, key is test_row_0/B:col10/1732224235510/Put/seqid=0 2024-11-21T21:23:56,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741841_1017 (size=12001) 2024-11-21T21:23:56,590 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b96b12779101462fbda0b3dddeb79d8e 2024-11-21T21:23:56,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/415d4916fc6849649cfd16ec0f2b6011 is 50, key is test_row_0/C:col10/1732224235510/Put/seqid=0 2024-11-21T21:23:56,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741842_1018 (size=12001) 2024-11-21T21:23:56,643 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/415d4916fc6849649cfd16ec0f2b6011 2024-11-21T21:23:56,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0f3e41bd59154355af4fd1666ada9c81 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0f3e41bd59154355af4fd1666ada9c81 2024-11-21T21:23:56,678 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0f3e41bd59154355af4fd1666ada9c81, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T21:23:56,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b96b12779101462fbda0b3dddeb79d8e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b12779101462fbda0b3dddeb79d8e 2024-11-21T21:23:56,698 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b12779101462fbda0b3dddeb79d8e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T21:23:56,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/415d4916fc6849649cfd16ec0f2b6011 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/415d4916fc6849649cfd16ec0f2b6011 2024-11-21T21:23:56,719 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/415d4916fc6849649cfd16ec0f2b6011, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T21:23:56,723 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 4b5690ae4b378a5d396cbe15e83372d1 in 228ms, sequenceid=38, compaction requested=false 2024-11-21T21:23:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T21:23:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T21:23:56,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T21:23:56,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4490 sec 2024-11-21T21:23:56,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4760 sec 2024-11-21T21:23:56,846 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:23:56,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:23:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:56,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/d6e45ce7ddf64caeb25f7b3eba724828 is 50, key is test_row_0/A:col10/1732224236841/Put/seqid=0 2024-11-21T21:23:56,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741843_1019 (size=9657) 2024-11-21T21:23:56,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/d6e45ce7ddf64caeb25f7b3eba724828 2024-11-21T21:23:56,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/65d7ced47467447087a3aee4636efbe8 is 50, key is test_row_0/B:col10/1732224236841/Put/seqid=0 2024-11-21T21:23:56,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224296958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224296961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224296965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224296969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224296975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741844_1020 (size=9657) 2024-11-21T21:23:56,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/65d7ced47467447087a3aee4636efbe8 2024-11-21T21:23:57,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/254d1758c9f140cc98f93fb2dc2aed3b is 50, key is test_row_0/C:col10/1732224236841/Put/seqid=0 2024-11-21T21:23:57,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741845_1021 (size=9657) 2024-11-21T21:23:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224297090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224297090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224297091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224297091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224297100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224297300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224297301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224297311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224297311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224297321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:23:57,420 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:23:57,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:23:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T21:23:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:23:57,426 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:23:57,428 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:23:57,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:23:57,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/254d1758c9f140cc98f93fb2dc2aed3b 2024-11-21T21:23:57,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/d6e45ce7ddf64caeb25f7b3eba724828 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/d6e45ce7ddf64caeb25f7b3eba724828 2024-11-21T21:23:57,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/d6e45ce7ddf64caeb25f7b3eba724828, entries=100, sequenceid=50, filesize=9.4 K 2024-11-21T21:23:57,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/65d7ced47467447087a3aee4636efbe8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/65d7ced47467447087a3aee4636efbe8 2024-11-21T21:23:57,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/65d7ced47467447087a3aee4636efbe8, entries=100, sequenceid=50, filesize=9.4 K 2024-11-21T21:23:57,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/254d1758c9f140cc98f93fb2dc2aed3b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/254d1758c9f140cc98f93fb2dc2aed3b 2024-11-21T21:23:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:23:57,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/254d1758c9f140cc98f93fb2dc2aed3b, entries=100, sequenceid=50, filesize=9.4 K 2024-11-21T21:23:57,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4b5690ae4b378a5d396cbe15e83372d1 in 698ms, sequenceid=50, compaction requested=true 2024-11-21T21:23:57,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:57,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:23:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:23:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:23:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:23:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:23:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:23:57,552 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:23:57,558 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:23:57,560 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:23:57,561 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:23:57,562 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:57,563 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/8748bf12e7fe4333bd88aedc7b5808b7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0f3e41bd59154355af4fd1666ada9c81, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/d6e45ce7ddf64caeb25f7b3eba724828] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=37.4 K 2024-11-21T21:23:57,564 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:23:57,564 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:23:57,564 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:57,565 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2480f351344f47aaa997080548b24ba6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b12779101462fbda0b3dddeb79d8e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/65d7ced47467447087a3aee4636efbe8] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=32.9 K 2024-11-21T21:23:57,566 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2480f351344f47aaa997080548b24ba6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732224235292 2024-11-21T21:23:57,567 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b96b12779101462fbda0b3dddeb79d8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732224235510 2024-11-21T21:23:57,568 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8748bf12e7fe4333bd88aedc7b5808b7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732224235292 2024-11-21T21:23:57,569 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f3e41bd59154355af4fd1666ada9c81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732224235510 2024-11-21T21:23:57,570 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d7ced47467447087a3aee4636efbe8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732224236841 2024-11-21T21:23:57,571 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6e45ce7ddf64caeb25f7b3eba724828, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732224236841 2024-11-21T21:23:57,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T21:23:57,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:57,584 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:23:57,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e6b76cddeee34be8b09dd644eea8b815 is 50, key is test_row_0/A:col10/1732224236961/Put/seqid=0 2024-11-21T21:23:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:57,653 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#11 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:23:57,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:57,654 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#10 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:23:57,655 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/df449c51ed1844ef93f6c570e8a9718a is 50, key is test_row_0/A:col10/1732224236841/Put/seqid=0 2024-11-21T21:23:57,655 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/1ae27e902bea4ac5b04ef4a61c80ecca is 50, key is test_row_0/B:col10/1732224236841/Put/seqid=0 2024-11-21T21:23:57,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741846_1022 (size=12001) 2024-11-21T21:23:57,664 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e6b76cddeee34be8b09dd644eea8b815 2024-11-21T21:23:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224297688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224297691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224297694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224297699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224297699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/af4cdcb4161e482f9803785bce649fa8 is 50, key is test_row_0/B:col10/1732224236961/Put/seqid=0 2024-11-21T21:23:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741847_1023 (size=12104) 2024-11-21T21:23:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:23:57,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741848_1024 (size=12104) 2024-11-21T21:23:57,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741849_1025 (size=12001) 2024-11-21T21:23:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224297812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224297811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224297813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224297812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:57,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224297820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224298030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224298031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224298030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224298040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224298040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:23:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,148 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/1ae27e902bea4ac5b04ef4a61c80ecca as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/1ae27e902bea4ac5b04ef4a61c80ecca 2024-11-21T21:23:58,184 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into 1ae27e902bea4ac5b04ef4a61c80ecca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:23:58,184 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:58,185 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/df449c51ed1844ef93f6c570e8a9718a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df449c51ed1844ef93f6c570e8a9718a 2024-11-21T21:23:58,187 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=13, startTime=1732224237551; duration=0sec 2024-11-21T21:23:58,193 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:23:58,193 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:23:58,194 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:23:58,197 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:23:58,197 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:23:58,197 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:58,197 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8662333747ae497bb8c8ef3b76d3e54e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/415d4916fc6849649cfd16ec0f2b6011, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/254d1758c9f140cc98f93fb2dc2aed3b] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=32.9 K 2024-11-21T21:23:58,199 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8662333747ae497bb8c8ef3b76d3e54e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732224235292 2024-11-21T21:23:58,201 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 415d4916fc6849649cfd16ec0f2b6011, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732224235510 2024-11-21T21:23:58,201 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/af4cdcb4161e482f9803785bce649fa8 2024-11-21T21:23:58,203 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 254d1758c9f140cc98f93fb2dc2aed3b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732224236841 2024-11-21T21:23:58,214 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into df449c51ed1844ef93f6c570e8a9718a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:23:58,214 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:58,215 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=13, startTime=1732224237547; duration=0sec 2024-11-21T21:23:58,215 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:23:58,215 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:23:58,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c92ad39476b04bd2a3f4a14b7f09a173 is 50, key is test_row_0/C:col10/1732224236961/Put/seqid=0 2024-11-21T21:23:58,232 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#14 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:23:58,233 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/a485b9942d644a18ba12011913529dbb is 50, key is test_row_0/C:col10/1732224236841/Put/seqid=0 2024-11-21T21:23:58,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741851_1027 (size=12104) 2024-11-21T21:23:58,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741850_1026 (size=12001) 2024-11-21T21:23:58,314 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/a485b9942d644a18ba12011913529dbb as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/a485b9942d644a18ba12011913529dbb 2024-11-21T21:23:58,351 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into a485b9942d644a18ba12011913529dbb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:23:58,351 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:58,351 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=13, startTime=1732224237551; duration=0sec 2024-11-21T21:23:58,355 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:23:58,355 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:23:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224298340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224298350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224298353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224298360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224298360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T21:23:58,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T21:23:58,379 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:23:58,379 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T21:23:58,380 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T21:23:58,380 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T21:23:58,382 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:23:58,382 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T21:23:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:23:58,705 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c92ad39476b04bd2a3f4a14b7f09a173 2024-11-21T21:23:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e6b76cddeee34be8b09dd644eea8b815 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e6b76cddeee34be8b09dd644eea8b815 2024-11-21T21:23:58,735 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e6b76cddeee34be8b09dd644eea8b815, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T21:23:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/af4cdcb4161e482f9803785bce649fa8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/af4cdcb4161e482f9803785bce649fa8 2024-11-21T21:23:58,761 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/af4cdcb4161e482f9803785bce649fa8, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T21:23:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c92ad39476b04bd2a3f4a14b7f09a173 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c92ad39476b04bd2a3f4a14b7f09a173 2024-11-21T21:23:58,785 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c92ad39476b04bd2a3f4a14b7f09a173, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T21:23:58,788 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 4b5690ae4b378a5d396cbe15e83372d1 in 1205ms, sequenceid=74, compaction requested=false 2024-11-21T21:23:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T21:23:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T21:23:58,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T21:23:58,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3630 sec 2024-11-21T21:23:58,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3760 sec 2024-11-21T21:23:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:58,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:23:58,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:23:58,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/fe536449a24c4e139f319b661f4b6d0a is 50, key is test_row_0/A:col10/1732224238860/Put/seqid=0 2024-11-21T21:23:58,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741852_1028 (size=14341) 2024-11-21T21:23:58,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/fe536449a24c4e139f319b661f4b6d0a 2024-11-21T21:23:58,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224298916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224298926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224298930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224298934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224298925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:58,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/6a405279f29f49a5b9da7fb84105167a is 50, key is test_row_0/B:col10/1732224238860/Put/seqid=0 2024-11-21T21:23:58,963 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:23:59,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741853_1029 (size=12001) 2024-11-21T21:23:59,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/6a405279f29f49a5b9da7fb84105167a 2024-11-21T21:23:59,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c3b2870bd33241e5af08987cbf20f598 is 50, key is test_row_0/C:col10/1732224238860/Put/seqid=0 2024-11-21T21:23:59,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224299040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224299051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224299050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224299051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224299051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,065 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T21:23:59,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741854_1030 (size=12001) 2024-11-21T21:23:59,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c3b2870bd33241e5af08987cbf20f598 2024-11-21T21:23:59,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/fe536449a24c4e139f319b661f4b6d0a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fe536449a24c4e139f319b661f4b6d0a 2024-11-21T21:23:59,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fe536449a24c4e139f319b661f4b6d0a, entries=200, sequenceid=91, filesize=14.0 K 2024-11-21T21:23:59,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/6a405279f29f49a5b9da7fb84105167a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/6a405279f29f49a5b9da7fb84105167a 2024-11-21T21:23:59,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/6a405279f29f49a5b9da7fb84105167a, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T21:23:59,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c3b2870bd33241e5af08987cbf20f598 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c3b2870bd33241e5af08987cbf20f598 2024-11-21T21:23:59,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c3b2870bd33241e5af08987cbf20f598, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T21:23:59,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 4b5690ae4b378a5d396cbe15e83372d1 in 290ms, sequenceid=91, compaction requested=true 2024-11-21T21:23:59,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:59,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:23:59,156 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:23:59,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:23:59,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:23:59,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:23:59,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:23:59,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:23:59,157 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:23:59,158 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:23:59,158 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:23:59,158 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:59,158 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df449c51ed1844ef93f6c570e8a9718a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e6b76cddeee34be8b09dd644eea8b815, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fe536449a24c4e139f319b661f4b6d0a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=37.5 K 2024-11-21T21:23:59,159 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting df449c51ed1844ef93f6c570e8a9718a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732224235580 2024-11-21T21:23:59,161 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b76cddeee34be8b09dd644eea8b815, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732224236955 2024-11-21T21:23:59,161 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:23:59,161 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:23:59,161 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:59,162 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/1ae27e902bea4ac5b04ef4a61c80ecca, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/af4cdcb4161e482f9803785bce649fa8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/6a405279f29f49a5b9da7fb84105167a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=35.3 K 2024-11-21T21:23:59,165 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting fe536449a24c4e139f319b661f4b6d0a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732224237687 2024-11-21T21:23:59,165 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae27e902bea4ac5b04ef4a61c80ecca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732224235580 2024-11-21T21:23:59,167 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting af4cdcb4161e482f9803785bce649fa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732224236955 2024-11-21T21:23:59,168 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a405279f29f49a5b9da7fb84105167a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732224237687 2024-11-21T21:23:59,195 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:23:59,196 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/41eb15c1f71743bfa8b4135c8f318c7c is 50, key is test_row_0/A:col10/1732224238860/Put/seqid=0 2024-11-21T21:23:59,196 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:23:59,197 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/20f6233f1e0249a0a5d22d32da0c8671 is 50, key is test_row_0/B:col10/1732224238860/Put/seqid=0 2024-11-21T21:23:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741855_1031 (size=12207) 2024-11-21T21:23:59,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:23:59,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:23:59,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741856_1032 (size=12207) 2024-11-21T21:23:59,261 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/41eb15c1f71743bfa8b4135c8f318c7c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/41eb15c1f71743bfa8b4135c8f318c7c 2024-11-21T21:23:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:59,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/dc821f4a585c4c92adab0039d013933f is 50, key is test_row_0/A:col10/1732224238924/Put/seqid=0 2024-11-21T21:23:59,277 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into 41eb15c1f71743bfa8b4135c8f318c7c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:23:59,277 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:59,277 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=13, startTime=1732224239155; duration=0sec 2024-11-21T21:23:59,278 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:23:59,278 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:23:59,278 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:23:59,281 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:23:59,281 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:23:59,281 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:59,281 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/a485b9942d644a18ba12011913529dbb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c92ad39476b04bd2a3f4a14b7f09a173, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c3b2870bd33241e5af08987cbf20f598] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=35.3 K 2024-11-21T21:23:59,283 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a485b9942d644a18ba12011913529dbb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732224235580 2024-11-21T21:23:59,284 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c92ad39476b04bd2a3f4a14b7f09a173, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732224236955 2024-11-21T21:23:59,285 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b2870bd33241e5af08987cbf20f598, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732224237687 2024-11-21T21:23:59,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741857_1033 (size=14341) 2024-11-21T21:23:59,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/dc821f4a585c4c92adab0039d013933f 2024-11-21T21:23:59,302 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:23:59,303 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/f2c9efd0bef74fde8a1c165ca4976c28 is 50, key is test_row_0/C:col10/1732224238860/Put/seqid=0 2024-11-21T21:23:59,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224299300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224299298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224299301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224299304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224299304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741858_1034 (size=12207) 2024-11-21T21:23:59,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/8f81f427183347198bcfaaf6acb76b82 is 50, key is test_row_0/B:col10/1732224238924/Put/seqid=0 2024-11-21T21:23:59,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741859_1035 (size=12001) 2024-11-21T21:23:59,346 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/f2c9efd0bef74fde8a1c165ca4976c28 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f2c9efd0bef74fde8a1c165ca4976c28 2024-11-21T21:23:59,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/8f81f427183347198bcfaaf6acb76b82 2024-11-21T21:23:59,363 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into f2c9efd0bef74fde8a1c165ca4976c28(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:23:59,363 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:59,363 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=13, startTime=1732224239156; duration=0sec 2024-11-21T21:23:59,363 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:23:59,364 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:23:59,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/5cf60c31f6504271a2d6d77246b4eeea is 50, key is test_row_0/C:col10/1732224238924/Put/seqid=0 2024-11-21T21:23:59,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741860_1036 (size=12001) 2024-11-21T21:23:59,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/5cf60c31f6504271a2d6d77246b4eeea 2024-11-21T21:23:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/dc821f4a585c4c92adab0039d013933f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/dc821f4a585c4c92adab0039d013933f 2024-11-21T21:23:59,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224299421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/dc821f4a585c4c92adab0039d013933f, entries=200, sequenceid=114, filesize=14.0 K 2024-11-21T21:23:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/8f81f427183347198bcfaaf6acb76b82 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8f81f427183347198bcfaaf6acb76b82 2024-11-21T21:23:59,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224299421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224299423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224299421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224299423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8f81f427183347198bcfaaf6acb76b82, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:23:59,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/5cf60c31f6504271a2d6d77246b4eeea as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5cf60c31f6504271a2d6d77246b4eeea 2024-11-21T21:23:59,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5cf60c31f6504271a2d6d77246b4eeea, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:23:59,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4b5690ae4b378a5d396cbe15e83372d1 in 191ms, sequenceid=114, compaction requested=false 2024-11-21T21:23:59,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:23:59,570 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:23:59,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:23:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T21:23:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:23:59,574 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:23:59,576 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:23:59,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:23:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:23:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:23:59,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T21:23:59,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/9f5a8c4fa28e42bebd08e6700fc2d243 is 50, key is test_row_0/A:col10/1732224239633/Put/seqid=0 2024-11-21T21:23:59,680 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/20f6233f1e0249a0a5d22d32da0c8671 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/20f6233f1e0249a0a5d22d32da0c8671 2024-11-21T21:23:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:23:59,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224299678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224299681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224299684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741861_1037 (size=14491) 2024-11-21T21:23:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224299684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224299689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,698 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into 20f6233f1e0249a0a5d22d32da0c8671(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:23:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,698 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:23:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,698 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=13, startTime=1732224239156; duration=0sec 2024-11-21T21:23:59,698 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:23:59,698 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:23:59,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:23:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:59,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:59,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224299807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224299807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224299807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224299811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:23:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224299812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:23:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:23:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:23:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:23:59,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:23:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:23:59,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:23:59,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:59,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:23:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224300020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224300030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224300031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224300031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224300031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:24:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:00,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/9f5a8c4fa28e42bebd08e6700fc2d243 2024-11-21T21:24:00,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/3c45f635903d4a5cb797a90891f12ac5 is 50, key is test_row_0/B:col10/1732224239633/Put/seqid=0 2024-11-21T21:24:00,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741862_1038 (size=12101) 2024-11-21T21:24:00,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/3c45f635903d4a5cb797a90891f12ac5 2024-11-21T21:24:00,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c05a1193a7ec48d1afc1f2069efa8f2c is 50, key is test_row_0/C:col10/1732224239633/Put/seqid=0 2024-11-21T21:24:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741863_1039 (size=12101) 2024-11-21T21:24:00,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c05a1193a7ec48d1afc1f2069efa8f2c 2024-11-21T21:24:00,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/9f5a8c4fa28e42bebd08e6700fc2d243 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9f5a8c4fa28e42bebd08e6700fc2d243 2024-11-21T21:24:00,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9f5a8c4fa28e42bebd08e6700fc2d243, entries=200, sequenceid=132, filesize=14.2 K 2024-11-21T21:24:00,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/3c45f635903d4a5cb797a90891f12ac5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c45f635903d4a5cb797a90891f12ac5 2024-11-21T21:24:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:24:00,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:24:00,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:00,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c45f635903d4a5cb797a90891f12ac5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-21T21:24:00,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c05a1193a7ec48d1afc1f2069efa8f2c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c05a1193a7ec48d1afc1f2069efa8f2c 2024-11-21T21:24:00,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c05a1193a7ec48d1afc1f2069efa8f2c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-21T21:24:00,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 4b5690ae4b378a5d396cbe15e83372d1 in 594ms, sequenceid=132, compaction requested=true 2024-11-21T21:24:00,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:00,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:00,229 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:00,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:00,229 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:00,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:00,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:00,232 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,232 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/41eb15c1f71743bfa8b4135c8f318c7c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/dc821f4a585c4c92adab0039d013933f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9f5a8c4fa28e42bebd08e6700fc2d243] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=40.1 K 2024-11-21T21:24:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:00,234 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:00,234 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:00,234 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,235 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/20f6233f1e0249a0a5d22d32da0c8671, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8f81f427183347198bcfaaf6acb76b82, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c45f635903d4a5cb797a90891f12ac5] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=35.5 K 2024-11-21T21:24:00,235 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41eb15c1f71743bfa8b4135c8f318c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732224237687 2024-11-21T21:24:00,236 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f6233f1e0249a0a5d22d32da0c8671, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732224237687 2024-11-21T21:24:00,236 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc821f4a585c4c92adab0039d013933f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224238908 2024-11-21T21:24:00,237 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f81f427183347198bcfaaf6acb76b82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224238908 2024-11-21T21:24:00,237 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f5a8c4fa28e42bebd08e6700fc2d243, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732224239299 2024-11-21T21:24:00,239 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c45f635903d4a5cb797a90891f12ac5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732224239299 2024-11-21T21:24:00,256 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:00,257 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/f14dd8708164483096139fe4b91765c9 is 50, key is test_row_0/A:col10/1732224239633/Put/seqid=0 2024-11-21T21:24:00,259 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:00,259 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/daca3cffe045491db84d02406e508309 is 50, key is test_row_0/B:col10/1732224239633/Put/seqid=0 2024-11-21T21:24:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741864_1040 (size=12409) 2024-11-21T21:24:00,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741865_1041 (size=12409) 2024-11-21T21:24:00,300 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/f14dd8708164483096139fe4b91765c9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f14dd8708164483096139fe4b91765c9 2024-11-21T21:24:00,302 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/daca3cffe045491db84d02406e508309 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/daca3cffe045491db84d02406e508309 2024-11-21T21:24:00,315 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into daca3cffe045491db84d02406e508309(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:00,315 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:00,315 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=13, startTime=1732224240229; duration=0sec 2024-11-21T21:24:00,316 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:00,316 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:00,316 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:00,318 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into f14dd8708164483096139fe4b91765c9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:00,318 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:00,318 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=13, startTime=1732224240229; duration=0sec 2024-11-21T21:24:00,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:00,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:00,320 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:00,320 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:00,320 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,320 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f2c9efd0bef74fde8a1c165ca4976c28, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5cf60c31f6504271a2d6d77246b4eeea, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c05a1193a7ec48d1afc1f2069efa8f2c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=35.5 K 2024-11-21T21:24:00,324 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c9efd0bef74fde8a1c165ca4976c28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732224237687 2024-11-21T21:24:00,326 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf60c31f6504271a2d6d77246b4eeea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224238908 2024-11-21T21:24:00,327 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c05a1193a7ec48d1afc1f2069efa8f2c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732224239299 2024-11-21T21:24:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:00,333 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:00,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T21:24:00,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/c18de8bd18d045f98c0823fae9b14ce2 is 50, key is test_row_0/A:col10/1732224239680/Put/seqid=0 2024-11-21T21:24:00,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:24:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224300358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,369 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#30 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:00,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224300363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,370 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/8cb387a1cc824cf188ce148b96e477ae is 50, key is test_row_0/C:col10/1732224239633/Put/seqid=0 2024-11-21T21:24:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224300364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224300366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224300367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741866_1042 (size=14541) 2024-11-21T21:24:00,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741867_1043 (size=12409) 2024-11-21T21:24:00,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224300481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224300481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224300482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224300482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224300482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:24:00,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:00,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:24:00,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:00,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224300690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224300689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224300690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224300690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224300690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:24:00,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/c18de8bd18d045f98c0823fae9b14ce2 2024-11-21T21:24:00,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/f646a7d1e379493ca8d3f2d697805862 is 50, key is test_row_0/B:col10/1732224239680/Put/seqid=0 2024-11-21T21:24:00,805 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/8cb387a1cc824cf188ce148b96e477ae as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8cb387a1cc824cf188ce148b96e477ae 2024-11-21T21:24:00,820 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into 8cb387a1cc824cf188ce148b96e477ae(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:00,820 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:00,820 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=13, startTime=1732224240234; duration=0sec 2024-11-21T21:24:00,820 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:00,820 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:00,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:24:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:00,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:00,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741868_1044 (size=12151) 2024-11-21T21:24:00,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/f646a7d1e379493ca8d3f2d697805862 2024-11-21T21:24:00,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/232b8daafc84475d81c5ec2dac278c6c is 50, key is test_row_0/C:col10/1732224239680/Put/seqid=0 2024-11-21T21:24:00,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741869_1045 (size=12151) 2024-11-21T21:24:00,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/232b8daafc84475d81c5ec2dac278c6c 2024-11-21T21:24:00,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/c18de8bd18d045f98c0823fae9b14ce2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/c18de8bd18d045f98c0823fae9b14ce2 2024-11-21T21:24:00,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/c18de8bd18d045f98c0823fae9b14ce2, entries=200, sequenceid=157, filesize=14.2 K 2024-11-21T21:24:00,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/f646a7d1e379493ca8d3f2d697805862 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f646a7d1e379493ca8d3f2d697805862 2024-11-21T21:24:00,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f646a7d1e379493ca8d3f2d697805862, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T21:24:00,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/232b8daafc84475d81c5ec2dac278c6c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/232b8daafc84475d81c5ec2dac278c6c 2024-11-21T21:24:00,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/232b8daafc84475d81c5ec2dac278c6c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T21:24:00,943 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T21:24:00,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 4b5690ae4b378a5d396cbe15e83372d1 in 611ms, sequenceid=157, compaction requested=false 2024-11-21T21:24:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:00,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:24:00,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:00,978 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:24:00,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e15c8104ebeb4929b3660a1c4745cf87 is 50, key is test_row_0/A:col10/1732224240360/Put/seqid=0 2024-11-21T21:24:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:01,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:01,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741870_1046 (size=12151) 2024-11-21T21:24:01,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224301037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224301039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224301044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224301046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224301046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224301149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224301150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224301160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224301161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224301160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224301360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224301360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224301370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224301370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224301371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,414 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e15c8104ebeb4929b3660a1c4745cf87 2024-11-21T21:24:01,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/f24adfa1c4ee43b1aa4127e20f12b7b9 is 50, key is test_row_0/B:col10/1732224240360/Put/seqid=0 2024-11-21T21:24:01,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741871_1047 (size=12151) 2024-11-21T21:24:01,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224301670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224301670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224301680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224301680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224301680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:24:01,844 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/f24adfa1c4ee43b1aa4127e20f12b7b9 2024-11-21T21:24:01,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bb6361488ad440a7b4e0fe546691b6ff is 50, key is test_row_0/C:col10/1732224240360/Put/seqid=0 2024-11-21T21:24:01,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741872_1048 (size=12151) 2024-11-21T21:24:01,882 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bb6361488ad440a7b4e0fe546691b6ff 2024-11-21T21:24:01,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e15c8104ebeb4929b3660a1c4745cf87 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e15c8104ebeb4929b3660a1c4745cf87 2024-11-21T21:24:01,898 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e15c8104ebeb4929b3660a1c4745cf87, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T21:24:01,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/f24adfa1c4ee43b1aa4127e20f12b7b9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f24adfa1c4ee43b1aa4127e20f12b7b9 2024-11-21T21:24:01,909 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f24adfa1c4ee43b1aa4127e20f12b7b9, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T21:24:01,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bb6361488ad440a7b4e0fe546691b6ff as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bb6361488ad440a7b4e0fe546691b6ff 2024-11-21T21:24:01,920 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bb6361488ad440a7b4e0fe546691b6ff, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T21:24:01,923 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 4b5690ae4b378a5d396cbe15e83372d1 in 945ms, sequenceid=172, compaction requested=true 2024-11-21T21:24:01,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:01,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:01,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T21:24:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T21:24:01,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T21:24:01,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3490 sec 2024-11-21T21:24:01,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3580 sec 2024-11-21T21:24:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:02,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:02,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:24:02,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/01f552bc8e99462197040780516d1583 is 50, key is test_row_0/A:col10/1732224241039/Put/seqid=0 2024-11-21T21:24:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224302199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224302201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224302201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224302203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224302206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741873_1049 (size=14541) 2024-11-21T21:24:02,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/01f552bc8e99462197040780516d1583 2024-11-21T21:24:02,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2277d796b9f84c2b86f4c36e6ba0ea21 is 50, key is test_row_0/B:col10/1732224241039/Put/seqid=0 2024-11-21T21:24:02,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741874_1050 (size=12151) 2024-11-21T21:24:02,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2277d796b9f84c2b86f4c36e6ba0ea21 2024-11-21T21:24:02,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/66a0643ef84345808e7fea142ccc18f1 is 50, key is test_row_0/C:col10/1732224241039/Put/seqid=0 2024-11-21T21:24:02,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741875_1051 (size=12151) 2024-11-21T21:24:02,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/66a0643ef84345808e7fea142ccc18f1 2024-11-21T21:24:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224302309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224302310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224302310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/01f552bc8e99462197040780516d1583 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/01f552bc8e99462197040780516d1583 2024-11-21T21:24:02,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224302320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224302320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/01f552bc8e99462197040780516d1583, entries=200, sequenceid=195, filesize=14.2 K 2024-11-21T21:24:02,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2277d796b9f84c2b86f4c36e6ba0ea21 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2277d796b9f84c2b86f4c36e6ba0ea21 2024-11-21T21:24:02,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2277d796b9f84c2b86f4c36e6ba0ea21, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T21:24:02,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/66a0643ef84345808e7fea142ccc18f1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/66a0643ef84345808e7fea142ccc18f1 2024-11-21T21:24:02,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/66a0643ef84345808e7fea142ccc18f1, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T21:24:02,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 4b5690ae4b378a5d396cbe15e83372d1 in 166ms, sequenceid=195, compaction requested=true 2024-11-21T21:24:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:02,349 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:02,349 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:02,351 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:02,351 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:02,351 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:02,351 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:02,351 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:02,351 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:02,351 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f14dd8708164483096139fe4b91765c9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/c18de8bd18d045f98c0823fae9b14ce2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e15c8104ebeb4929b3660a1c4745cf87, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/01f552bc8e99462197040780516d1583] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=52.4 K 2024-11-21T21:24:02,351 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/daca3cffe045491db84d02406e508309, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f646a7d1e379493ca8d3f2d697805862, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f24adfa1c4ee43b1aa4127e20f12b7b9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2277d796b9f84c2b86f4c36e6ba0ea21] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=47.7 K 2024-11-21T21:24:02,355 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f14dd8708164483096139fe4b91765c9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732224239299 2024-11-21T21:24:02,355 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting daca3cffe045491db84d02406e508309, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732224239299 2024-11-21T21:24:02,356 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18de8bd18d045f98c0823fae9b14ce2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732224239680 2024-11-21T21:24:02,356 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f646a7d1e379493ca8d3f2d697805862, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732224239680 2024-11-21T21:24:02,356 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15c8104ebeb4929b3660a1c4745cf87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732224240360 2024-11-21T21:24:02,356 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f24adfa1c4ee43b1aa4127e20f12b7b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732224240360 2024-11-21T21:24:02,357 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01f552bc8e99462197040780516d1583, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732224241015 2024-11-21T21:24:02,357 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2277d796b9f84c2b86f4c36e6ba0ea21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732224241015 2024-11-21T21:24:02,378 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:02,379 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/706eea3edf6e49efa608a7a8f2ce3418 is 50, key is test_row_0/B:col10/1732224241039/Put/seqid=0 2024-11-21T21:24:02,383 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:02,384 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/17b38e0ff2084831b6ff9e6771f3a4ac is 50, key is test_row_0/A:col10/1732224241039/Put/seqid=0 2024-11-21T21:24:02,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741876_1052 (size=12595) 2024-11-21T21:24:02,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741877_1053 (size=12595) 2024-11-21T21:24:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:02,521 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:02,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T21:24:02,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/62474ba9b6684db2879b9fd9a93e0628 is 50, key is test_row_0/A:col10/1732224242520/Put/seqid=0 2024-11-21T21:24:02,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741878_1054 (size=16931) 2024-11-21T21:24:02,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/62474ba9b6684db2879b9fd9a93e0628 2024-11-21T21:24:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224302546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224302547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224302548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224302551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224302554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/3c522d4f59574a9e88d6ebeed8fe68f1 is 50, key is test_row_0/B:col10/1732224242520/Put/seqid=0 2024-11-21T21:24:02,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741879_1055 (size=12151) 2024-11-21T21:24:02,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224302660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224302660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224302660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224302660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224302660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,805 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/706eea3edf6e49efa608a7a8f2ce3418 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/706eea3edf6e49efa608a7a8f2ce3418 2024-11-21T21:24:02,814 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into 706eea3edf6e49efa608a7a8f2ce3418(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:02,814 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:02,814 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=12, startTime=1732224242349; duration=0sec 2024-11-21T21:24:02,815 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:02,815 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:02,815 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:02,818 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:02,818 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:02,818 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:02,818 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8cb387a1cc824cf188ce148b96e477ae, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/232b8daafc84475d81c5ec2dac278c6c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bb6361488ad440a7b4e0fe546691b6ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/66a0643ef84345808e7fea142ccc18f1] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=47.7 K 2024-11-21T21:24:02,819 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb387a1cc824cf188ce148b96e477ae, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732224239299 2024-11-21T21:24:02,819 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 232b8daafc84475d81c5ec2dac278c6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732224239680 2024-11-21T21:24:02,819 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting bb6361488ad440a7b4e0fe546691b6ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732224240360 2024-11-21T21:24:02,820 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a0643ef84345808e7fea142ccc18f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732224241015 2024-11-21T21:24:02,831 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/17b38e0ff2084831b6ff9e6771f3a4ac as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/17b38e0ff2084831b6ff9e6771f3a4ac 2024-11-21T21:24:02,840 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into 17b38e0ff2084831b6ff9e6771f3a4ac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:02,841 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:02,841 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=12, startTime=1732224242349; duration=0sec 2024-11-21T21:24:02,841 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:02,841 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:02,841 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:02,842 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/f01e4493868246f5943fa8c5611757a5 is 50, key is test_row_0/C:col10/1732224241039/Put/seqid=0 2024-11-21T21:24:02,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741880_1056 (size=12595) 2024-11-21T21:24:02,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,872 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/f01e4493868246f5943fa8c5611757a5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f01e4493868246f5943fa8c5611757a5 2024-11-21T21:24:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224302869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224302870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224302870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224302870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224302870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:02,896 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into f01e4493868246f5943fa8c5611757a5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:02,896 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:02,897 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=12, startTime=1732224242349; duration=0sec 2024-11-21T21:24:02,897 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:02,897 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:02,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/3c522d4f59574a9e88d6ebeed8fe68f1 2024-11-21T21:24:03,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/2533439cf7ee4c298db4e405548527fd is 50, key is test_row_0/C:col10/1732224242520/Put/seqid=0 2024-11-21T21:24:03,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741881_1057 (size=12151) 2024-11-21T21:24:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224303179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224303179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224303180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224303180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224303180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/2533439cf7ee4c298db4e405548527fd 2024-11-21T21:24:03,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/62474ba9b6684db2879b9fd9a93e0628 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/62474ba9b6684db2879b9fd9a93e0628 2024-11-21T21:24:03,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/62474ba9b6684db2879b9fd9a93e0628, entries=250, sequenceid=211, filesize=16.5 K 2024-11-21T21:24:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/3c522d4f59574a9e88d6ebeed8fe68f1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c522d4f59574a9e88d6ebeed8fe68f1 2024-11-21T21:24:03,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c522d4f59574a9e88d6ebeed8fe68f1, entries=150, sequenceid=211, filesize=11.9 K 2024-11-21T21:24:03,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/2533439cf7ee4c298db4e405548527fd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/2533439cf7ee4c298db4e405548527fd 2024-11-21T21:24:03,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/2533439cf7ee4c298db4e405548527fd, entries=150, sequenceid=211, filesize=11.9 K 2024-11-21T21:24:03,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 4b5690ae4b378a5d396cbe15e83372d1 in 928ms, sequenceid=211, compaction requested=false 2024-11-21T21:24:03,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:03,693 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:03,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:24:03,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/990fee8879184ccb8063374824b03e1c is 50, key is test_row_0/A:col10/1732224242545/Put/seqid=0 2024-11-21T21:24:03,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224303706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741882_1058 (size=16931) 2024-11-21T21:24:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224303707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224303708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224303709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224303710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:24:03,729 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:03,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T21:24:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:24:03,733 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:03,734 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:03,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224303820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224303820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224303820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224303820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224303820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:24:03,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:24:03,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:03,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:03,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:03,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224304030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224304030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224304030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224304030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224304030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:24:04,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/990fee8879184ccb8063374824b03e1c 2024-11-21T21:24:04,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/cc6f0faa77cc4078a606914f0a449205 is 50, key is test_row_0/B:col10/1732224242545/Put/seqid=0 2024-11-21T21:24:04,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:24:04,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:04,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:04,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741883_1059 (size=12151) 2024-11-21T21:24:04,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/cc6f0faa77cc4078a606914f0a449205 2024-11-21T21:24:04,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bf94a1430b714406a919d6ab39775fce is 50, key is test_row_0/C:col10/1732224242545/Put/seqid=0 2024-11-21T21:24:04,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741884_1060 (size=12151) 2024-11-21T21:24:04,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bf94a1430b714406a919d6ab39775fce 2024-11-21T21:24:04,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/990fee8879184ccb8063374824b03e1c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/990fee8879184ccb8063374824b03e1c 2024-11-21T21:24:04,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/990fee8879184ccb8063374824b03e1c, entries=250, sequenceid=238, filesize=16.5 K 2024-11-21T21:24:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/cc6f0faa77cc4078a606914f0a449205 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/cc6f0faa77cc4078a606914f0a449205 2024-11-21T21:24:04,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/cc6f0faa77cc4078a606914f0a449205, entries=150, sequenceid=238, filesize=11.9 K 2024-11-21T21:24:04,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bf94a1430b714406a919d6ab39775fce as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf94a1430b714406a919d6ab39775fce 2024-11-21T21:24:04,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf94a1430b714406a919d6ab39775fce, entries=150, sequenceid=238, filesize=11.9 K 2024-11-21T21:24:04,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 4b5690ae4b378a5d396cbe15e83372d1 in 541ms, sequenceid=238, compaction requested=true 2024-11-21T21:24:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:04,234 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:04,234 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:04,236 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 46457 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:04,236 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:04,236 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:04,236 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:04,236 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:04,236 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/17b38e0ff2084831b6ff9e6771f3a4ac, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/62474ba9b6684db2879b9fd9a93e0628, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/990fee8879184ccb8063374824b03e1c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=45.4 K 2024-11-21T21:24:04,236 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:04,237 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/706eea3edf6e49efa608a7a8f2ce3418, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c522d4f59574a9e88d6ebeed8fe68f1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/cc6f0faa77cc4078a606914f0a449205] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=36.0 K 2024-11-21T21:24:04,237 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 706eea3edf6e49efa608a7a8f2ce3418, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732224241015 2024-11-21T21:24:04,238 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b38e0ff2084831b6ff9e6771f3a4ac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732224241015 2024-11-21T21:24:04,238 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c522d4f59574a9e88d6ebeed8fe68f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732224242201 2024-11-21T21:24:04,238 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62474ba9b6684db2879b9fd9a93e0628, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732224242199 2024-11-21T21:24:04,238 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6f0faa77cc4078a606914f0a449205, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732224242545 2024-11-21T21:24:04,239 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 990fee8879184ccb8063374824b03e1c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732224242545 2024-11-21T21:24:04,250 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:04,251 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/df2f6a23377f4bed86a76385d785a636 is 50, key is test_row_0/A:col10/1732224242545/Put/seqid=0 2024-11-21T21:24:04,252 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:04,253 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/fb8afaa49473478cb1536b98061f271d is 50, key is test_row_0/B:col10/1732224242545/Put/seqid=0 2024-11-21T21:24:04,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:24:04,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:04,281 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:24:04,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/7a93c60a8c7a4f82903eb910456ad583 is 50, key is test_row_0/A:col10/1732224243707/Put/seqid=0 2024-11-21T21:24:04,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741885_1061 (size=12697) 2024-11-21T21:24:04,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741886_1062 (size=12697) 2024-11-21T21:24:04,307 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/fb8afaa49473478cb1536b98061f271d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/fb8afaa49473478cb1536b98061f271d 2024-11-21T21:24:04,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741887_1063 (size=12151) 2024-11-21T21:24:04,319 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/7a93c60a8c7a4f82903eb910456ad583 2024-11-21T21:24:04,323 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into fb8afaa49473478cb1536b98061f271d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:04,324 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:04,324 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=13, startTime=1732224244234; duration=0sec 2024-11-21T21:24:04,325 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:04,326 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:04,326 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:04,332 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:04,333 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:04,333 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:04,333 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f01e4493868246f5943fa8c5611757a5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/2533439cf7ee4c298db4e405548527fd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf94a1430b714406a919d6ab39775fce] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=36.0 K 2024-11-21T21:24:04,334 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f01e4493868246f5943fa8c5611757a5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732224241015 2024-11-21T21:24:04,335 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2533439cf7ee4c298db4e405548527fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732224242201 2024-11-21T21:24:04,335 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting bf94a1430b714406a919d6ab39775fce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732224242545 2024-11-21T21:24:04,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/5768c9cf2ae84d8583ca5e0312c7ad86 is 50, key is test_row_0/B:col10/1732224243707/Put/seqid=0 2024-11-21T21:24:04,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:04,350 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:04,351 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/5f381f32e1ad47c6b7e60fc7c4997d5d is 50, key is test_row_0/C:col10/1732224242545/Put/seqid=0 2024-11-21T21:24:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741888_1064 (size=12151) 2024-11-21T21:24:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:24:04,360 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/5768c9cf2ae84d8583ca5e0312c7ad86 2024-11-21T21:24:04,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741889_1065 (size=12697) 2024-11-21T21:24:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/facb8d71314a41a698de8b04a7dbf15a is 50, key is test_row_0/C:col10/1732224243707/Put/seqid=0 2024-11-21T21:24:04,385 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/5f381f32e1ad47c6b7e60fc7c4997d5d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5f381f32e1ad47c6b7e60fc7c4997d5d 2024-11-21T21:24:04,394 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into 5f381f32e1ad47c6b7e60fc7c4997d5d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:04,395 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:04,395 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=13, startTime=1732224244235; duration=0sec 2024-11-21T21:24:04,395 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:04,395 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:04,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224304397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224304398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224304399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224304405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741890_1066 (size=12151) 2024-11-21T21:24:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224304407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224304510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224304511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224304511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224304520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224304520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,705 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/df2f6a23377f4bed86a76385d785a636 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df2f6a23377f4bed86a76385d785a636 2024-11-21T21:24:04,713 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into df2f6a23377f4bed86a76385d785a636(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:04,713 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:04,714 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=13, startTime=1732224244234; duration=0sec 2024-11-21T21:24:04,714 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:04,714 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224304720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224304720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224304720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224304730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224304730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:04,812 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/facb8d71314a41a698de8b04a7dbf15a 2024-11-21T21:24:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/7a93c60a8c7a4f82903eb910456ad583 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/7a93c60a8c7a4f82903eb910456ad583 2024-11-21T21:24:04,835 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/7a93c60a8c7a4f82903eb910456ad583, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T21:24:04,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/5768c9cf2ae84d8583ca5e0312c7ad86 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/5768c9cf2ae84d8583ca5e0312c7ad86 2024-11-21T21:24:04,852 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/5768c9cf2ae84d8583ca5e0312c7ad86, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T21:24:04,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/facb8d71314a41a698de8b04a7dbf15a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/facb8d71314a41a698de8b04a7dbf15a 2024-11-21T21:24:04,862 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/facb8d71314a41a698de8b04a7dbf15a, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T21:24:04,864 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4b5690ae4b378a5d396cbe15e83372d1 in 583ms, sequenceid=251, compaction requested=false 2024-11-21T21:24:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T21:24:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T21:24:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:24:04,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T21:24:04,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1350 sec 2024-11-21T21:24:04,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1450 sec 2024-11-21T21:24:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:05,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:05,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T21:24:05,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/9635f4c640674be8b1d6b15a26485960 is 50, key is test_row_0/A:col10/1732224244403/Put/seqid=0 2024-11-21T21:24:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224305040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224305040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224305041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224305044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224305045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741891_1067 (size=12301) 2024-11-21T21:24:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224305150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224305150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224305159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224305360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224305360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224305369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/9635f4c640674be8b1d6b15a26485960 2024-11-21T21:24:05,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/9099786b16584809b764777af842266e is 50, key is test_row_0/B:col10/1732224244403/Put/seqid=0 2024-11-21T21:24:05,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741892_1068 (size=12301) 2024-11-21T21:24:05,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/9099786b16584809b764777af842266e 2024-11-21T21:24:05,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0 is 50, key is test_row_0/C:col10/1732224244403/Put/seqid=0 2024-11-21T21:24:05,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741893_1069 (size=12301) 2024-11-21T21:24:05,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224305550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224305550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224305670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224305670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224305679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:24:05,880 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:05,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T21:24:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:24:05,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:05,884 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:05,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:05,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0 2024-11-21T21:24:05,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/9635f4c640674be8b1d6b15a26485960 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9635f4c640674be8b1d6b15a26485960 2024-11-21T21:24:05,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9635f4c640674be8b1d6b15a26485960, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T21:24:05,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/9099786b16584809b764777af842266e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/9099786b16584809b764777af842266e 2024-11-21T21:24:05,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/9099786b16584809b764777af842266e, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T21:24:05,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0 2024-11-21T21:24:05,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T21:24:05,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 4b5690ae4b378a5d396cbe15e83372d1 in 930ms, sequenceid=280, compaction requested=true 2024-11-21T21:24:05,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:05,963 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:05,963 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:05,964 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:05,964 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:05,964 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:05,964 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/fb8afaa49473478cb1536b98061f271d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/5768c9cf2ae84d8583ca5e0312c7ad86, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/9099786b16584809b764777af842266e] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=36.3 K 2024-11-21T21:24:05,965 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:05,965 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:05,965 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:05,965 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df2f6a23377f4bed86a76385d785a636, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/7a93c60a8c7a4f82903eb910456ad583, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9635f4c640674be8b1d6b15a26485960] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=36.3 K 2024-11-21T21:24:05,965 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting fb8afaa49473478cb1536b98061f271d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732224242545 2024-11-21T21:24:05,966 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5768c9cf2ae84d8583ca5e0312c7ad86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224243704 2024-11-21T21:24:05,966 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting df2f6a23377f4bed86a76385d785a636, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732224242545 2024-11-21T21:24:05,966 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9099786b16584809b764777af842266e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732224244402 2024-11-21T21:24:05,966 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a93c60a8c7a4f82903eb910456ad583, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224243704 2024-11-21T21:24:05,966 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9635f4c640674be8b1d6b15a26485960, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732224244402 2024-11-21T21:24:05,978 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:05,979 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/b12feab5f4e748a8b6cefdea8c66f839 is 50, key is test_row_0/A:col10/1732224244403/Put/seqid=0 2024-11-21T21:24:05,985 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:05,986 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b96b3cd3575248b1a9dfdd2ae2fcd0dd is 50, key is test_row_0/B:col10/1732224244403/Put/seqid=0 2024-11-21T21:24:05,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741894_1070 (size=12949) 2024-11-21T21:24:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:24:06,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741895_1071 (size=12949) 2024-11-21T21:24:06,003 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/b12feab5f4e748a8b6cefdea8c66f839 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b12feab5f4e748a8b6cefdea8c66f839 2024-11-21T21:24:06,014 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into b12feab5f4e748a8b6cefdea8c66f839(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:06,014 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:06,014 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=13, startTime=1732224245963; duration=0sec 2024-11-21T21:24:06,014 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:06,014 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:06,015 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:06,016 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:06,016 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:06,016 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:06,017 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5f381f32e1ad47c6b7e60fc7c4997d5d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/facb8d71314a41a698de8b04a7dbf15a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=36.3 K 2024-11-21T21:24:06,017 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f381f32e1ad47c6b7e60fc7c4997d5d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732224242545 2024-11-21T21:24:06,017 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting facb8d71314a41a698de8b04a7dbf15a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224243704 2024-11-21T21:24:06,018 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c09a9e8c97bd4e3aa7a2c1d63804d2d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732224244402 2024-11-21T21:24:06,028 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:06,028 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/019e09232806409b98a8f067d96c706e is 50, key is test_row_0/C:col10/1732224244403/Put/seqid=0 2024-11-21T21:24:06,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T21:24:06,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:06,036 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:24:06,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741896_1072 (size=12949) 2024-11-21T21:24:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6f70efaa1bcb47b996be0819e7115e3c is 50, key is test_row_0/A:col10/1732224245042/Put/seqid=0 2024-11-21T21:24:06,051 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/019e09232806409b98a8f067d96c706e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/019e09232806409b98a8f067d96c706e 2024-11-21T21:24:06,060 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into 019e09232806409b98a8f067d96c706e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:06,060 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:06,061 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=13, startTime=1732224245963; duration=0sec 2024-11-21T21:24:06,061 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:06,061 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741897_1073 (size=12301) 2024-11-21T21:24:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:06,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:24:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224306225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224306231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224306237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224306339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224306350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224306350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,414 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b96b3cd3575248b1a9dfdd2ae2fcd0dd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b3cd3575248b1a9dfdd2ae2fcd0dd 2024-11-21T21:24:06,424 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into b96b3cd3575248b1a9dfdd2ae2fcd0dd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:06,424 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:06,424 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=13, startTime=1732224245963; duration=0sec 2024-11-21T21:24:06,425 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:06,425 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:06,464 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6f70efaa1bcb47b996be0819e7115e3c 2024-11-21T21:24:06,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b74105a54ea9498d8cdf4a3cf2522f4b is 50, key is test_row_0/B:col10/1732224245042/Put/seqid=0 2024-11-21T21:24:06,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741898_1074 (size=12301) 2024-11-21T21:24:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:24:06,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224306552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224306560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224306560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224306560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,566 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 4b5690ae4b378a5d396cbe15e83372d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224306569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,573 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 4b5690ae4b378a5d396cbe15e83372d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224306860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224306870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224306870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:06,901 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b74105a54ea9498d8cdf4a3cf2522f4b 2024-11-21T21:24:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c2870f385c924c888af2ba2215d17c2e is 50, key is test_row_0/C:col10/1732224245042/Put/seqid=0 2024-11-21T21:24:06,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741899_1075 (size=12301) 2024-11-21T21:24:06,916 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c2870f385c924c888af2ba2215d17c2e 2024-11-21T21:24:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6f70efaa1bcb47b996be0819e7115e3c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6f70efaa1bcb47b996be0819e7115e3c 2024-11-21T21:24:06,932 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6f70efaa1bcb47b996be0819e7115e3c, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T21:24:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b74105a54ea9498d8cdf4a3cf2522f4b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b74105a54ea9498d8cdf4a3cf2522f4b 2024-11-21T21:24:06,942 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b74105a54ea9498d8cdf4a3cf2522f4b, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T21:24:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c2870f385c924c888af2ba2215d17c2e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c2870f385c924c888af2ba2215d17c2e 2024-11-21T21:24:06,955 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c2870f385c924c888af2ba2215d17c2e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T21:24:06,956 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 4b5690ae4b378a5d396cbe15e83372d1 in 920ms, sequenceid=292, compaction requested=false 2024-11-21T21:24:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:06,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T21:24:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T21:24:06,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T21:24:06,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-21T21:24:06,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0830 sec 2024-11-21T21:24:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:24:07,030 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:07,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T21:24:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:24:07,040 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:07,042 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:07,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:24:07,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T21:24:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:07,197 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T21:24:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/b7dd9cce3cf342bcbd1ec81c84851286 is 50, key is test_row_0/A:col10/1732224246221/Put/seqid=0 2024-11-21T21:24:07,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741900_1076 (size=12301) 2024-11-21T21:24:07,230 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/b7dd9cce3cf342bcbd1ec81c84851286 2024-11-21T21:24:07,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/c313b720e7cd4a408a015cfb8e746a95 is 50, key is test_row_0/B:col10/1732224246221/Put/seqid=0 2024-11-21T21:24:07,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741901_1077 (size=12301) 2024-11-21T21:24:07,264 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/c313b720e7cd4a408a015cfb8e746a95 2024-11-21T21:24:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/39b5cb50c33642f49f4bfa6883d5c2f5 is 50, key is test_row_0/C:col10/1732224246221/Put/seqid=0 2024-11-21T21:24:07,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741902_1078 (size=12301) 2024-11-21T21:24:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:24:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:07,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224307386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224307390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224307391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224307499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224307499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224307500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:24:07,702 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/39b5cb50c33642f49f4bfa6883d5c2f5 2024-11-21T21:24:07,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/b7dd9cce3cf342bcbd1ec81c84851286 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b7dd9cce3cf342bcbd1ec81c84851286 2024-11-21T21:24:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224307710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224307710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,719 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b7dd9cce3cf342bcbd1ec81c84851286, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T21:24:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/c313b720e7cd4a408a015cfb8e746a95 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c313b720e7cd4a408a015cfb8e746a95 2024-11-21T21:24:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224307719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:07,727 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c313b720e7cd4a408a015cfb8e746a95, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T21:24:07,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-21T21:24:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/39b5cb50c33642f49f4bfa6883d5c2f5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/39b5cb50c33642f49f4bfa6883d5c2f5 2024-11-21T21:24:07,752 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/39b5cb50c33642f49f4bfa6883d5c2f5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T21:24:07,754 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 4b5690ae4b378a5d396cbe15e83372d1 in 556ms, sequenceid=320, compaction requested=true 2024-11-21T21:24:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T21:24:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T21:24:07,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T21:24:07,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 714 msec 2024-11-21T21:24:07,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 723 msec 2024-11-21T21:24:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:08,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:08,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:24:08,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/fa24b52473d240f795c7a9634712e88f is 50, key is test_row_0/A:col10/1732224248022/Put/seqid=0 2024-11-21T21:24:08,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741903_1079 (size=12301) 2024-11-21T21:24:08,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/fa24b52473d240f795c7a9634712e88f 2024-11-21T21:24:08,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/8d2eec21730043ad8cd38bde074559d2 is 50, key is test_row_0/B:col10/1732224248022/Put/seqid=0 2024-11-21T21:24:08,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224308126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224308130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224308130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741904_1080 (size=12301) 2024-11-21T21:24:08,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/8d2eec21730043ad8cd38bde074559d2 2024-11-21T21:24:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:24:08,181 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:08,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T21:24:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:24:08,191 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:08,193 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:08,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:08,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/489b4371d6004bbaaf1eccda9e8d1281 is 50, key is test_row_0/C:col10/1732224248022/Put/seqid=0 2024-11-21T21:24:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741905_1081 (size=12301) 2024-11-21T21:24:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224308240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224308240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224308240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:24:08,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:24:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224308450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224308452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224308460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:24:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:24:08,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224308569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,573 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 4b5690ae4b378a5d396cbe15e83372d1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224308579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,585 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 4b5690ae4b378a5d396cbe15e83372d1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/489b4371d6004bbaaf1eccda9e8d1281 2024-11-21T21:24:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/fa24b52473d240f795c7a9634712e88f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fa24b52473d240f795c7a9634712e88f 2024-11-21T21:24:08,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fa24b52473d240f795c7a9634712e88f, entries=150, sequenceid=334, filesize=12.0 K 2024-11-21T21:24:08,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/8d2eec21730043ad8cd38bde074559d2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8d2eec21730043ad8cd38bde074559d2 2024-11-21T21:24:08,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8d2eec21730043ad8cd38bde074559d2, entries=150, sequenceid=334, filesize=12.0 K 2024-11-21T21:24:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/489b4371d6004bbaaf1eccda9e8d1281 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/489b4371d6004bbaaf1eccda9e8d1281 2024-11-21T21:24:08,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:24:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:08,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/489b4371d6004bbaaf1eccda9e8d1281, entries=150, sequenceid=334, filesize=12.0 K 2024-11-21T21:24:08,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 4b5690ae4b378a5d396cbe15e83372d1 in 593ms, sequenceid=334, compaction requested=true 2024-11-21T21:24:08,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:08,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:08,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:08,665 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:08,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:08,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:08,666 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:08,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:08,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:08,667 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:08,667 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:08,667 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:08,667 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:08,667 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,667 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,668 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b3cd3575248b1a9dfdd2ae2fcd0dd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b74105a54ea9498d8cdf4a3cf2522f4b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c313b720e7cd4a408a015cfb8e746a95, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8d2eec21730043ad8cd38bde074559d2] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=48.7 K 2024-11-21T21:24:08,668 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b12feab5f4e748a8b6cefdea8c66f839, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6f70efaa1bcb47b996be0819e7115e3c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b7dd9cce3cf342bcbd1ec81c84851286, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fa24b52473d240f795c7a9634712e88f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=48.7 K 2024-11-21T21:24:08,668 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12feab5f4e748a8b6cefdea8c66f839, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732224244402 2024-11-21T21:24:08,668 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b96b3cd3575248b1a9dfdd2ae2fcd0dd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732224244402 2024-11-21T21:24:08,669 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b74105a54ea9498d8cdf4a3cf2522f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732224245038 2024-11-21T21:24:08,669 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f70efaa1bcb47b996be0819e7115e3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732224245038 2024-11-21T21:24:08,669 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c313b720e7cd4a408a015cfb8e746a95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732224246221 2024-11-21T21:24:08,669 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7dd9cce3cf342bcbd1ec81c84851286, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732224246221 2024-11-21T21:24:08,670 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2eec21730043ad8cd38bde074559d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224247388 2024-11-21T21:24:08,671 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa24b52473d240f795c7a9634712e88f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224247388 2024-11-21T21:24:08,694 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:08,694 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0c93d7b04e6a40b994c52bd2ce139cc8 is 50, key is test_row_0/A:col10/1732224248022/Put/seqid=0 2024-11-21T21:24:08,702 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:08,703 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/7304478046d84f529fcfc49f8a73b064 is 50, key is test_row_0/B:col10/1732224248022/Put/seqid=0 2024-11-21T21:24:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741906_1082 (size=13085) 2024-11-21T21:24:08,746 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0c93d7b04e6a40b994c52bd2ce139cc8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0c93d7b04e6a40b994c52bd2ce139cc8 2024-11-21T21:24:08,754 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into 0c93d7b04e6a40b994c52bd2ce139cc8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:08,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:08,754 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=12, startTime=1732224248665; duration=0sec 2024-11-21T21:24:08,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:08,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:08,755 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:08,760 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:08,760 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:08,760 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,760 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/019e09232806409b98a8f067d96c706e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c2870f385c924c888af2ba2215d17c2e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/39b5cb50c33642f49f4bfa6883d5c2f5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/489b4371d6004bbaaf1eccda9e8d1281] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=48.7 K 2024-11-21T21:24:08,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741907_1083 (size=13085) 2024-11-21T21:24:08,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:08,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:24:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:08,769 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019e09232806409b98a8f067d96c706e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732224244402 2024-11-21T21:24:08,770 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2870f385c924c888af2ba2215d17c2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732224245038 2024-11-21T21:24:08,771 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b5cb50c33642f49f4bfa6883d5c2f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732224246221 2024-11-21T21:24:08,772 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489b4371d6004bbaaf1eccda9e8d1281, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224247388 2024-11-21T21:24:08,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/46d6683fbe75465eb510965aa78e7687 is 50, key is test_row_0/A:col10/1732224248098/Put/seqid=0 2024-11-21T21:24:08,796 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:08,797 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/15a364539f4741dd99465c24b5771191 is 50, key is test_row_0/C:col10/1732224248022/Put/seqid=0 2024-11-21T21:24:08,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224308793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224308793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224308795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741908_1084 (size=12301) 2024-11-21T21:24:08,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/46d6683fbe75465eb510965aa78e7687 2024-11-21T21:24:08,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:24:08,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:08,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:24:08,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741909_1085 (size=13085) 2024-11-21T21:24:08,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/e434b37958ac446c992b713e31eeda9d is 50, key is test_row_0/B:col10/1732224248098/Put/seqid=0 2024-11-21T21:24:08,845 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/15a364539f4741dd99465c24b5771191 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/15a364539f4741dd99465c24b5771191 2024-11-21T21:24:08,859 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into 15a364539f4741dd99465c24b5771191(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:08,859 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:08,859 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=12, startTime=1732224248666; duration=0sec 2024-11-21T21:24:08,859 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:08,859 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:08,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741910_1086 (size=12301) 2024-11-21T21:24:08,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/e434b37958ac446c992b713e31eeda9d 2024-11-21T21:24:08,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/11892aa01bcf4b7c9c5059edfa5a7ea6 is 50, key is test_row_0/C:col10/1732224248098/Put/seqid=0 2024-11-21T21:24:08,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741911_1087 (size=12301) 2024-11-21T21:24:08,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/11892aa01bcf4b7c9c5059edfa5a7ea6 2024-11-21T21:24:08,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224308922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224308923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224308923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:08,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:24:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:08,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:08,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:08,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/46d6683fbe75465eb510965aa78e7687 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/46d6683fbe75465eb510965aa78e7687 2024-11-21T21:24:08,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/46d6683fbe75465eb510965aa78e7687, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T21:24:08,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/e434b37958ac446c992b713e31eeda9d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/e434b37958ac446c992b713e31eeda9d 2024-11-21T21:24:08,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/e434b37958ac446c992b713e31eeda9d, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T21:24:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/11892aa01bcf4b7c9c5059edfa5a7ea6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/11892aa01bcf4b7c9c5059edfa5a7ea6 2024-11-21T21:24:09,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/11892aa01bcf4b7c9c5059edfa5a7ea6, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T21:24:09,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4b5690ae4b378a5d396cbe15e83372d1 in 241ms, sequenceid=359, compaction requested=false 2024-11-21T21:24:09,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:09,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:24:09,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:09,124 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:24:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0d0143160c5946729ee59a3ac65c253e is 50, key is test_row_0/A:col10/1732224248789/Put/seqid=0 2024-11-21T21:24:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:09,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741912_1088 (size=12301) 2024-11-21T21:24:09,168 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/7304478046d84f529fcfc49f8a73b064 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7304478046d84f529fcfc49f8a73b064 2024-11-21T21:24:09,175 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into 7304478046d84f529fcfc49f8a73b064(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:09,175 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:09,175 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=12, startTime=1732224248666; duration=0sec 2024-11-21T21:24:09,176 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:09,176 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:09,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224309198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224309202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224309203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224309309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224309310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224309310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:24:09,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224309519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224309520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224309520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,541 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0d0143160c5946729ee59a3ac65c253e 2024-11-21T21:24:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/4b4c5bea9e8f45539b2efc5d992a50ba is 50, key is test_row_0/B:col10/1732224248789/Put/seqid=0 2024-11-21T21:24:09,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741913_1089 (size=12301) 2024-11-21T21:24:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224309829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224309829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224309829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:09,967 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/4b4c5bea9e8f45539b2efc5d992a50ba 2024-11-21T21:24:09,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c6921708f68a4ed2ab3ee52bd66cc493 is 50, key is test_row_0/C:col10/1732224248789/Put/seqid=0 2024-11-21T21:24:09,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741914_1090 (size=12301) 2024-11-21T21:24:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:24:10,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224310339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:10,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224310339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:10,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224310340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:10,381 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c6921708f68a4ed2ab3ee52bd66cc493 2024-11-21T21:24:10,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/0d0143160c5946729ee59a3ac65c253e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0d0143160c5946729ee59a3ac65c253e 2024-11-21T21:24:10,392 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0d0143160c5946729ee59a3ac65c253e, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T21:24:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/4b4c5bea9e8f45539b2efc5d992a50ba as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4b4c5bea9e8f45539b2efc5d992a50ba 2024-11-21T21:24:10,399 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4b4c5bea9e8f45539b2efc5d992a50ba, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T21:24:10,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/c6921708f68a4ed2ab3ee52bd66cc493 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c6921708f68a4ed2ab3ee52bd66cc493 2024-11-21T21:24:10,406 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c6921708f68a4ed2ab3ee52bd66cc493, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T21:24:10,407 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4b5690ae4b378a5d396cbe15e83372d1 in 1283ms, sequenceid=372, compaction requested=true 2024-11-21T21:24:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T21:24:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T21:24:10,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T21:24:10,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2160 sec 2024-11-21T21:24:10,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2300 sec 2024-11-21T21:24:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:11,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:24:11,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/bcaa31e20e5845388e64283565942a5e is 50, key is test_row_0/A:col10/1732224249197/Put/seqid=0 2024-11-21T21:24:11,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741915_1091 (size=14741) 2024-11-21T21:24:11,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/bcaa31e20e5845388e64283565942a5e 2024-11-21T21:24:11,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224311368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224311372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224311373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b6802d67f718442a98f04cf5ff540a9a is 50, key is test_row_0/B:col10/1732224249197/Put/seqid=0 2024-11-21T21:24:11,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741916_1092 (size=12301) 2024-11-21T21:24:11,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224311480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224311480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224311480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224311690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224311690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224311690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:11,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b6802d67f718442a98f04cf5ff540a9a 2024-11-21T21:24:11,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/eb9683c2e88449f1b83e35b524fc0168 is 50, key is test_row_0/C:col10/1732224249197/Put/seqid=0 2024-11-21T21:24:11,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741917_1093 (size=12301) 2024-11-21T21:24:11,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/eb9683c2e88449f1b83e35b524fc0168 2024-11-21T21:24:11,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/bcaa31e20e5845388e64283565942a5e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/bcaa31e20e5845388e64283565942a5e 2024-11-21T21:24:11,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/bcaa31e20e5845388e64283565942a5e, entries=200, sequenceid=398, filesize=14.4 K 2024-11-21T21:24:11,819 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T21:24:11,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/b6802d67f718442a98f04cf5ff540a9a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b6802d67f718442a98f04cf5ff540a9a 2024-11-21T21:24:11,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b6802d67f718442a98f04cf5ff540a9a, entries=150, sequenceid=398, filesize=12.0 K 2024-11-21T21:24:11,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/eb9683c2e88449f1b83e35b524fc0168 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/eb9683c2e88449f1b83e35b524fc0168 2024-11-21T21:24:11,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/eb9683c2e88449f1b83e35b524fc0168, entries=150, sequenceid=398, filesize=12.0 K 2024-11-21T21:24:11,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 4b5690ae4b378a5d396cbe15e83372d1 in 487ms, sequenceid=398, compaction requested=true 2024-11-21T21:24:11,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:11,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:11,839 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:11,839 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:11,843 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52428 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:11,843 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:11,843 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:11,843 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:11,843 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:11,843 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:11,843 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0c93d7b04e6a40b994c52bd2ce139cc8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/46d6683fbe75465eb510965aa78e7687, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0d0143160c5946729ee59a3ac65c253e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/bcaa31e20e5845388e64283565942a5e] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=51.2 K 2024-11-21T21:24:11,843 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7304478046d84f529fcfc49f8a73b064, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/e434b37958ac446c992b713e31eeda9d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4b4c5bea9e8f45539b2efc5d992a50ba, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b6802d67f718442a98f04cf5ff540a9a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=48.8 K 2024-11-21T21:24:11,843 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c93d7b04e6a40b994c52bd2ce139cc8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224247388 2024-11-21T21:24:11,843 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7304478046d84f529fcfc49f8a73b064, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224247388 2024-11-21T21:24:11,844 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d6683fbe75465eb510965aa78e7687, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732224248098 2024-11-21T21:24:11,845 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e434b37958ac446c992b713e31eeda9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732224248098 2024-11-21T21:24:11,845 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0143160c5946729ee59a3ac65c253e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732224248783 2024-11-21T21:24:11,845 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4c5bea9e8f45539b2efc5d992a50ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732224248783 2024-11-21T21:24:11,845 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcaa31e20e5845388e64283565942a5e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732224249197 2024-11-21T21:24:11,845 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b6802d67f718442a98f04cf5ff540a9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732224249197 2024-11-21T21:24:11,855 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:11,856 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2fee48dfce244fec89e4a643470d5a93 is 50, key is test_row_0/B:col10/1732224249197/Put/seqid=0 2024-11-21T21:24:11,860 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#82 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:11,861 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/f1083774c4884d928c641e3d10ad956b is 50, key is test_row_0/A:col10/1732224249197/Put/seqid=0 2024-11-21T21:24:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741918_1094 (size=13221) 2024-11-21T21:24:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741919_1095 (size=13221) 2024-11-21T21:24:11,885 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/f1083774c4884d928c641e3d10ad956b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f1083774c4884d928c641e3d10ad956b 2024-11-21T21:24:11,885 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/2fee48dfce244fec89e4a643470d5a93 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2fee48dfce244fec89e4a643470d5a93 2024-11-21T21:24:11,893 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into f1083774c4884d928c641e3d10ad956b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:11,893 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:11,893 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=12, startTime=1732224251838; duration=0sec 2024-11-21T21:24:11,894 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:11,894 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:11,894 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:11,895 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into 2fee48dfce244fec89e4a643470d5a93(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:11,895 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:11,895 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=12, startTime=1732224251839; duration=0sec 2024-11-21T21:24:11,896 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:11,896 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:11,896 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:11,896 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:11,896 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:11,897 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/15a364539f4741dd99465c24b5771191, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/11892aa01bcf4b7c9c5059edfa5a7ea6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c6921708f68a4ed2ab3ee52bd66cc493, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/eb9683c2e88449f1b83e35b524fc0168] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=48.8 K 2024-11-21T21:24:11,898 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a364539f4741dd99465c24b5771191, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224247388 2024-11-21T21:24:11,898 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11892aa01bcf4b7c9c5059edfa5a7ea6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732224248098 2024-11-21T21:24:11,899 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6921708f68a4ed2ab3ee52bd66cc493, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732224248783 2024-11-21T21:24:11,899 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb9683c2e88449f1b83e35b524fc0168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732224249197 2024-11-21T21:24:11,914 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:11,914 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/e7317410afee4ec0a78de8a5b339a68d is 50, key is test_row_0/C:col10/1732224249197/Put/seqid=0 2024-11-21T21:24:11,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741920_1096 (size=13221) 2024-11-21T21:24:11,936 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/e7317410afee4ec0a78de8a5b339a68d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e7317410afee4ec0a78de8a5b339a68d 2024-11-21T21:24:11,946 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into e7317410afee4ec0a78de8a5b339a68d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:11,946 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:11,946 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=12, startTime=1732224251839; duration=0sec 2024-11-21T21:24:11,946 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:11,946 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:12,001 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:12,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:24:12,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6a621320ffd142cabb19faa9f1c8908e is 50, key is test_row_0/A:col10/1732224251366/Put/seqid=0 2024-11-21T21:24:12,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741921_1097 (size=14741) 2024-11-21T21:24:12,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6a621320ffd142cabb19faa9f1c8908e 2024-11-21T21:24:12,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/4ce2a5cd768342c2b479f59439196969 is 50, key is test_row_0/B:col10/1732224251366/Put/seqid=0 2024-11-21T21:24:12,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224312034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224312038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224312040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741922_1098 (size=12301) 2024-11-21T21:24:12,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224312149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224312149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224312149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:24:12,350 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:12,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T21:24:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:24:12,355 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:12,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:12,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:12,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224312360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224312360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224312360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/4ce2a5cd768342c2b479f59439196969 2024-11-21T21:24:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:24:12,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bd4e8d567592404b967117bc17069ed0 is 50, key is test_row_0/C:col10/1732224251366/Put/seqid=0 2024-11-21T21:24:12,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741923_1099 (size=12301) 2024-11-21T21:24:12,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T21:24:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:12,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47152 deadline: 1732224312609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47142 deadline: 1732224312609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,610 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 4b5690ae4b378a5d396cbe15e83372d1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,612 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 4b5690ae4b378a5d396cbe15e83372d1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T21:24:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:12,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:24:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224312670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224312670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224312670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:12,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T21:24:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:12,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:12,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bd4e8d567592404b967117bc17069ed0 2024-11-21T21:24:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6a621320ffd142cabb19faa9f1c8908e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6a621320ffd142cabb19faa9f1c8908e 2024-11-21T21:24:12,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6a621320ffd142cabb19faa9f1c8908e, entries=200, sequenceid=416, filesize=14.4 K 2024-11-21T21:24:12,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/4ce2a5cd768342c2b479f59439196969 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4ce2a5cd768342c2b479f59439196969 2024-11-21T21:24:12,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4ce2a5cd768342c2b479f59439196969, entries=150, sequenceid=416, filesize=12.0 K 2024-11-21T21:24:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bd4e8d567592404b967117bc17069ed0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bd4e8d567592404b967117bc17069ed0 2024-11-21T21:24:12,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bd4e8d567592404b967117bc17069ed0, entries=150, sequenceid=416, filesize=12.0 K 2024-11-21T21:24:12,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4b5690ae4b378a5d396cbe15e83372d1 in 912ms, sequenceid=416, compaction requested=false 2024-11-21T21:24:12,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:12,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T21:24:12,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:12,972 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:24:12,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/37835cd2d6c4474cbd857b0b2a1eaa62 is 50, key is test_row_0/A:col10/1732224252012/Put/seqid=0 2024-11-21T21:24:12,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741924_1100 (size=12301) 2024-11-21T21:24:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:24:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224313193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224313194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224313195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224313300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224313310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224313310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,381 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/37835cd2d6c4474cbd857b0b2a1eaa62 2024-11-21T21:24:13,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/66da125a74e84039be4d43011dea7dcc is 50, key is test_row_0/B:col10/1732224252012/Put/seqid=0 2024-11-21T21:24:13,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741925_1101 (size=12301) 2024-11-21T21:24:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:24:13,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224313510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224313519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224313519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,794 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/66da125a74e84039be4d43011dea7dcc 2024-11-21T21:24:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bf3dade3b9aa4a20ab6a33e503e10969 is 50, key is test_row_0/C:col10/1732224252012/Put/seqid=0 2024-11-21T21:24:13,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741926_1102 (size=12301) 2024-11-21T21:24:13,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224313819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224313829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224313830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,209 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bf3dade3b9aa4a20ab6a33e503e10969 2024-11-21T21:24:14,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/37835cd2d6c4474cbd857b0b2a1eaa62 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/37835cd2d6c4474cbd857b0b2a1eaa62 2024-11-21T21:24:14,225 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/37835cd2d6c4474cbd857b0b2a1eaa62, entries=150, sequenceid=437, filesize=12.0 K 2024-11-21T21:24:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/66da125a74e84039be4d43011dea7dcc as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/66da125a74e84039be4d43011dea7dcc 2024-11-21T21:24:14,234 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/66da125a74e84039be4d43011dea7dcc, entries=150, sequenceid=437, filesize=12.0 K 2024-11-21T21:24:14,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/bf3dade3b9aa4a20ab6a33e503e10969 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf3dade3b9aa4a20ab6a33e503e10969 2024-11-21T21:24:14,241 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf3dade3b9aa4a20ab6a33e503e10969, entries=150, sequenceid=437, filesize=12.0 K 2024-11-21T21:24:14,243 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 4b5690ae4b378a5d396cbe15e83372d1 in 1271ms, sequenceid=437, compaction requested=true 2024-11-21T21:24:14,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:14,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T21:24:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T21:24:14,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T21:24:14,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8880 sec 2024-11-21T21:24:14,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8970 sec 2024-11-21T21:24:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:14,331 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:14,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T21:24:14,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6e55661ce3084967bc7a4810aee766c4 is 50, key is test_row_0/A:col10/1732224254329/Put/seqid=0 2024-11-21T21:24:14,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741927_1103 (size=14741) 2024-11-21T21:24:14,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6e55661ce3084967bc7a4810aee766c4 2024-11-21T21:24:14,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/ad5e601bea44488c8a2ab420ead4892d is 50, key is test_row_0/B:col10/1732224254329/Put/seqid=0 2024-11-21T21:24:14,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741928_1104 (size=12301) 2024-11-21T21:24:14,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/ad5e601bea44488c8a2ab420ead4892d 2024-11-21T21:24:14,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224314381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224314383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224314383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/dbc1db4358cf43b7a0f8f60fa169f63c is 50, key is test_row_0/C:col10/1732224254329/Put/seqid=0 2024-11-21T21:24:14,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741929_1105 (size=12301) 2024-11-21T21:24:14,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/dbc1db4358cf43b7a0f8f60fa169f63c 2024-11-21T21:24:14,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/6e55661ce3084967bc7a4810aee766c4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6e55661ce3084967bc7a4810aee766c4 2024-11-21T21:24:14,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6e55661ce3084967bc7a4810aee766c4, entries=200, sequenceid=453, filesize=14.4 K 2024-11-21T21:24:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/ad5e601bea44488c8a2ab420ead4892d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ad5e601bea44488c8a2ab420ead4892d 2024-11-21T21:24:14,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ad5e601bea44488c8a2ab420ead4892d, entries=150, sequenceid=453, filesize=12.0 K 2024-11-21T21:24:14,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/dbc1db4358cf43b7a0f8f60fa169f63c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/dbc1db4358cf43b7a0f8f60fa169f63c 2024-11-21T21:24:14,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/dbc1db4358cf43b7a0f8f60fa169f63c, entries=150, sequenceid=453, filesize=12.0 K 2024-11-21T21:24:14,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 4b5690ae4b378a5d396cbe15e83372d1 in 109ms, sequenceid=453, compaction requested=true 2024-11-21T21:24:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:14,440 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:14,440 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:14,442 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:14,442 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:14,442 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,443 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2fee48dfce244fec89e4a643470d5a93, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4ce2a5cd768342c2b479f59439196969, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/66da125a74e84039be4d43011dea7dcc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ad5e601bea44488c8a2ab420ead4892d] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=48.9 K 2024-11-21T21:24:14,443 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:14,443 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:14,443 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,443 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f1083774c4884d928c641e3d10ad956b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6a621320ffd142cabb19faa9f1c8908e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/37835cd2d6c4474cbd857b0b2a1eaa62, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6e55661ce3084967bc7a4810aee766c4] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=53.7 K 2024-11-21T21:24:14,443 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fee48dfce244fec89e4a643470d5a93, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732224249197 2024-11-21T21:24:14,443 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1083774c4884d928c641e3d10ad956b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732224249197 2024-11-21T21:24:14,443 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce2a5cd768342c2b479f59439196969, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732224251366 2024-11-21T21:24:14,444 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 66da125a74e84039be4d43011dea7dcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732224252012 2024-11-21T21:24:14,444 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a621320ffd142cabb19faa9f1c8908e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732224251366 2024-11-21T21:24:14,444 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37835cd2d6c4474cbd857b0b2a1eaa62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732224252012 2024-11-21T21:24:14,444 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ad5e601bea44488c8a2ab420ead4892d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732224253192 2024-11-21T21:24:14,444 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e55661ce3084967bc7a4810aee766c4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732224253185 2024-11-21T21:24:14,453 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:14,454 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:14,454 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/07e1e5582bbd419e95807cff6a3fe5d4 is 50, key is test_row_0/B:col10/1732224254329/Put/seqid=0 2024-11-21T21:24:14,455 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/2b92096753344697978b912196f9ba5f is 50, key is test_row_0/A:col10/1732224254329/Put/seqid=0 2024-11-21T21:24:14,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741930_1106 (size=13357) 2024-11-21T21:24:14,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741931_1107 (size=13357) 2024-11-21T21:24:14,481 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/07e1e5582bbd419e95807cff6a3fe5d4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/07e1e5582bbd419e95807cff6a3fe5d4 2024-11-21T21:24:14,490 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into 07e1e5582bbd419e95807cff6a3fe5d4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:14,491 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:14,491 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=12, startTime=1732224254440; duration=0sec 2024-11-21T21:24:14,491 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:14,491 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:14,492 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:14,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:14,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T21:24:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:14,494 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:14,494 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:14,494 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,494 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e7317410afee4ec0a78de8a5b339a68d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bd4e8d567592404b967117bc17069ed0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf3dade3b9aa4a20ab6a33e503e10969, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/dbc1db4358cf43b7a0f8f60fa169f63c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=48.9 K 2024-11-21T21:24:14,495 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e7317410afee4ec0a78de8a5b339a68d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732224249197 2024-11-21T21:24:14,496 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting bd4e8d567592404b967117bc17069ed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732224251366 2024-11-21T21:24:14,497 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3dade3b9aa4a20ab6a33e503e10969, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732224252012 2024-11-21T21:24:14,497 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting dbc1db4358cf43b7a0f8f60fa169f63c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732224253192 2024-11-21T21:24:14,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/225499324ff44b90b19102086112135d is 50, key is test_row_0/A:col10/1732224254491/Put/seqid=0 2024-11-21T21:24:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:24:14,500 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:14,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T21:24:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:24:14,503 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:14,506 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:14,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:14,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741932_1108 (size=17181) 2024-11-21T21:24:14,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224314516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/225499324ff44b90b19102086112135d 2024-11-21T21:24:14,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224314518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224314518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,529 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:14,530 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/d30787d21ebf4e2cb52ec522fb516911 is 50, key is test_row_0/C:col10/1732224254329/Put/seqid=0 2024-11-21T21:24:14,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/7be4d6cabb18440da96bd37fd50d0c1c is 50, key is test_row_0/B:col10/1732224254491/Put/seqid=0 2024-11-21T21:24:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741934_1110 (size=12301) 2024-11-21T21:24:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741933_1109 (size=13357) 2024-11-21T21:24:14,544 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/d30787d21ebf4e2cb52ec522fb516911 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/d30787d21ebf4e2cb52ec522fb516911 2024-11-21T21:24:14,554 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into d30787d21ebf4e2cb52ec522fb516911(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:14,554 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:14,554 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=12, startTime=1732224254440; duration=0sec 2024-11-21T21:24:14,555 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:14,555 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:24:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224314631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224314631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224314631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:24:14,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:24:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:24:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224314840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224314840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224314849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:14,872 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/2b92096753344697978b912196f9ba5f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/2b92096753344697978b912196f9ba5f 2024-11-21T21:24:14,880 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into 2b92096753344697978b912196f9ba5f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:14,880 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:14,880 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=12, startTime=1732224254440; duration=0sec 2024-11-21T21:24:14,880 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:14,880 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:14,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/7be4d6cabb18440da96bd37fd50d0c1c 2024-11-21T21:24:14,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/1e2226c38b314ba6890516db339bbe57 is 50, key is test_row_0/C:col10/1732224254491/Put/seqid=0 2024-11-21T21:24:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741935_1111 (size=12301) 2024-11-21T21:24:14,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/1e2226c38b314ba6890516db339bbe57 2024-11-21T21:24:14,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/225499324ff44b90b19102086112135d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/225499324ff44b90b19102086112135d 2024-11-21T21:24:14,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:24:14,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:14,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:14,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:14,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/225499324ff44b90b19102086112135d, entries=250, sequenceid=477, filesize=16.8 K 2024-11-21T21:24:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/7be4d6cabb18440da96bd37fd50d0c1c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7be4d6cabb18440da96bd37fd50d0c1c 2024-11-21T21:24:14,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7be4d6cabb18440da96bd37fd50d0c1c, entries=150, sequenceid=477, filesize=12.0 K 2024-11-21T21:24:14,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/1e2226c38b314ba6890516db339bbe57 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1e2226c38b314ba6890516db339bbe57 2024-11-21T21:24:14,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1e2226c38b314ba6890516db339bbe57, entries=150, sequenceid=477, filesize=12.0 K 2024-11-21T21:24:14,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 4b5690ae4b378a5d396cbe15e83372d1 in 505ms, sequenceid=477, compaction requested=false 2024-11-21T21:24:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:15,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:24:15,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:15,123 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:24:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/607f70f62a2b4a54a14b56948b417ade is 50, key is test_row_0/A:col10/1732224254507/Put/seqid=0 2024-11-21T21:24:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:24:15,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741936_1112 (size=12301) 2024-11-21T21:24:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:15,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. as already flushing 2024-11-21T21:24:15,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224315191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224315202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224315202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,256 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T21:24:15,256 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:15,256 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T21:24:15,257 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:15,258 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T21:24:15,258 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:15,260 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,260 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,260 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,261 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,261 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,261 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:15,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:15,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:15,264 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T21:24:15,264 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:15,264 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,264 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:15,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:15,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224315310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224315310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224315310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224315519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224315519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224315519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,539 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/607f70f62a2b4a54a14b56948b417ade 2024-11-21T21:24:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/c3185614559a425bb650d93a8df7a67a is 50, key is test_row_0/B:col10/1732224254507/Put/seqid=0 2024-11-21T21:24:15,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741937_1113 (size=12301) 2024-11-21T21:24:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:24:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224315830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224315831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224315831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:15,954 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/c3185614559a425bb650d93a8df7a67a 2024-11-21T21:24:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/1fa4fde9ad504a12869760b546c96570 is 50, key is test_row_0/C:col10/1732224254507/Put/seqid=0 2024-11-21T21:24:15,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741938_1114 (size=12301) 2024-11-21T21:24:16,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:16,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:16,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47114 deadline: 1732224316340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47082 deadline: 1732224316339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47160 deadline: 1732224316339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:24:16,373 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/1fa4fde9ad504a12869760b546c96570 2024-11-21T21:24:16,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/607f70f62a2b4a54a14b56948b417ade as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/607f70f62a2b4a54a14b56948b417ade 2024-11-21T21:24:16,382 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/607f70f62a2b4a54a14b56948b417ade, entries=150, sequenceid=493, filesize=12.0 K 2024-11-21T21:24:16,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/c3185614559a425bb650d93a8df7a67a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c3185614559a425bb650d93a8df7a67a 2024-11-21T21:24:16,388 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c3185614559a425bb650d93a8df7a67a, entries=150, sequenceid=493, filesize=12.0 K 2024-11-21T21:24:16,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/1fa4fde9ad504a12869760b546c96570 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1fa4fde9ad504a12869760b546c96570 2024-11-21T21:24:16,393 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1fa4fde9ad504a12869760b546c96570, entries=150, sequenceid=493, filesize=12.0 K 2024-11-21T21:24:16,394 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 4b5690ae4b378a5d396cbe15e83372d1 in 1271ms, sequenceid=493, compaction requested=true 2024-11-21T21:24:16,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:16,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:16,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T21:24:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T21:24:16,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T21:24:16,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8900 sec 2024-11-21T21:24:16,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8970 sec 2024-11-21T21:24:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:24:16,651 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:17,139 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:24:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:17,353 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:17,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T21:24:17,353 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T21:24:17,353 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T21:24:17,353 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T21:24:17,353 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:17,353 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:17,353 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:17,353 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:17,353 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:17,353 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:17,353 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:17,354 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:17,354 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:17,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:17,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:17,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:17,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e800aa60d70d4c2a88423adc265c8900 is 50, key is test_row_0/A:col10/1732224255182/Put/seqid=0 2024-11-21T21:24:17,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741939_1115 (size=12301) 2024-11-21T21:24:17,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e800aa60d70d4c2a88423adc265c8900 2024-11-21T21:24:17,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/ee625fcfd22140eb86a72e77b79add37 is 50, key is test_row_0/B:col10/1732224255182/Put/seqid=0 2024-11-21T21:24:17,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741940_1116 (size=12301) 2024-11-21T21:24:18,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/ee625fcfd22140eb86a72e77b79add37 2024-11-21T21:24:18,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/e3285349328d4d97b21846631e471e37 is 50, key is test_row_0/C:col10/1732224255182/Put/seqid=0 2024-11-21T21:24:18,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741941_1117 (size=12301) 2024-11-21T21:24:18,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/e3285349328d4d97b21846631e471e37 2024-11-21T21:24:18,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/e800aa60d70d4c2a88423adc265c8900 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e800aa60d70d4c2a88423adc265c8900 2024-11-21T21:24:18,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e800aa60d70d4c2a88423adc265c8900, entries=150, sequenceid=519, filesize=12.0 K 2024-11-21T21:24:18,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/ee625fcfd22140eb86a72e77b79add37 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ee625fcfd22140eb86a72e77b79add37 2024-11-21T21:24:18,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ee625fcfd22140eb86a72e77b79add37, entries=150, sequenceid=519, filesize=12.0 K 2024-11-21T21:24:18,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/e3285349328d4d97b21846631e471e37 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e3285349328d4d97b21846631e471e37 2024-11-21T21:24:18,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e3285349328d4d97b21846631e471e37, entries=150, sequenceid=519, filesize=12.0 K 2024-11-21T21:24:18,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 4b5690ae4b378a5d396cbe15e83372d1 in 1275ms, sequenceid=519, compaction requested=true 2024-11-21T21:24:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:18,628 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:18,628 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b5690ae4b378a5d396cbe15e83372d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:18,629 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:18,629 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55140 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:18,630 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/B is initiating minor compaction (all files) 2024-11-21T21:24:18,630 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/A is initiating minor compaction (all files) 2024-11-21T21:24:18,630 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/A in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:18,630 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/B in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:18,630 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/2b92096753344697978b912196f9ba5f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/225499324ff44b90b19102086112135d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/607f70f62a2b4a54a14b56948b417ade, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e800aa60d70d4c2a88423adc265c8900] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=53.8 K 2024-11-21T21:24:18,630 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/07e1e5582bbd419e95807cff6a3fe5d4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7be4d6cabb18440da96bd37fd50d0c1c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c3185614559a425bb650d93a8df7a67a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ee625fcfd22140eb86a72e77b79add37] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=49.1 K 2024-11-21T21:24:18,630 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e1e5582bbd419e95807cff6a3fe5d4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732224253192 2024-11-21T21:24:18,630 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b92096753344697978b912196f9ba5f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732224253192 2024-11-21T21:24:18,631 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be4d6cabb18440da96bd37fd50d0c1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732224254381 2024-11-21T21:24:18,631 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 225499324ff44b90b19102086112135d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732224254369 2024-11-21T21:24:18,631 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c3185614559a425bb650d93a8df7a67a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732224254507 2024-11-21T21:24:18,631 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607f70f62a2b4a54a14b56948b417ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732224254507 2024-11-21T21:24:18,631 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ee625fcfd22140eb86a72e77b79add37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732224255175 2024-11-21T21:24:18,631 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e800aa60d70d4c2a88423adc265c8900, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732224255175 2024-11-21T21:24:18,639 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:18,639 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:18,640 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/303c6c75e2694007adf33233b1f9cb4f is 50, key is test_row_0/B:col10/1732224255182/Put/seqid=0 2024-11-21T21:24:18,640 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/777ebe36e1754b6aa0502feaaa1329ba is 50, key is test_row_0/A:col10/1732224255182/Put/seqid=0 2024-11-21T21:24:18,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741943_1119 (size=13493) 2024-11-21T21:24:18,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741942_1118 (size=13493) 2024-11-21T21:24:19,068 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/777ebe36e1754b6aa0502feaaa1329ba as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/777ebe36e1754b6aa0502feaaa1329ba 2024-11-21T21:24:19,068 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/303c6c75e2694007adf33233b1f9cb4f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/303c6c75e2694007adf33233b1f9cb4f 2024-11-21T21:24:19,073 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/A of 4b5690ae4b378a5d396cbe15e83372d1 into 777ebe36e1754b6aa0502feaaa1329ba(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:19,073 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/B of 4b5690ae4b378a5d396cbe15e83372d1 into 303c6c75e2694007adf33233b1f9cb4f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:19,073 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:19,073 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:19,073 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/A, priority=12, startTime=1732224258628; duration=0sec 2024-11-21T21:24:19,073 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/B, priority=12, startTime=1732224258628; duration=0sec 2024-11-21T21:24:19,073 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:19,073 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:A 2024-11-21T21:24:19,073 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:19,073 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:B 2024-11-21T21:24:19,073 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:19,075 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:19,075 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 4b5690ae4b378a5d396cbe15e83372d1/C is initiating minor compaction (all files) 2024-11-21T21:24:19,075 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b5690ae4b378a5d396cbe15e83372d1/C in TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:19,075 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/d30787d21ebf4e2cb52ec522fb516911, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1e2226c38b314ba6890516db339bbe57, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1fa4fde9ad504a12869760b546c96570, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e3285349328d4d97b21846631e471e37] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp, totalSize=49.1 K 2024-11-21T21:24:19,075 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d30787d21ebf4e2cb52ec522fb516911, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732224253192 2024-11-21T21:24:19,076 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e2226c38b314ba6890516db339bbe57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732224254381 2024-11-21T21:24:19,076 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa4fde9ad504a12869760b546c96570, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732224254507 2024-11-21T21:24:19,076 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3285349328d4d97b21846631e471e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732224255175 2024-11-21T21:24:19,085 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b5690ae4b378a5d396cbe15e83372d1#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:19,086 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/620af1b948014decbc7a392733e1c52c is 50, key is test_row_0/C:col10/1732224255182/Put/seqid=0 2024-11-21T21:24:19,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741944_1120 (size=13493) 2024-11-21T21:24:19,503 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/620af1b948014decbc7a392733e1c52c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/620af1b948014decbc7a392733e1c52c 2024-11-21T21:24:19,511 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b5690ae4b378a5d396cbe15e83372d1/C of 4b5690ae4b378a5d396cbe15e83372d1 into 620af1b948014decbc7a392733e1c52c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:19,511 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b5690ae4b378a5d396cbe15e83372d1: 2024-11-21T21:24:19,511 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., storeName=4b5690ae4b378a5d396cbe15e83372d1/C, priority=12, startTime=1732224258628; duration=0sec 2024-11-21T21:24:19,511 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:19,511 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b5690ae4b378a5d396cbe15e83372d1:C 2024-11-21T21:24:22,647 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T21:24:22,647 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:22,647 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:22,647 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:22,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:22,715 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T21:24:22,715 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:22,715 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:22,716 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:22,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3497 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3462 2024-11-21T21:24:22,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:24:22,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1534 2024-11-21T21:24:22,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4594 rows 2024-11-21T21:24:22,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1550 2024-11-21T21:24:22,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4643 rows 2024-11-21T21:24:22,717 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:24:22,717 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:24:22,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:22,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:22,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:22,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:24:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:24:22,728 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224262727"}]},"ts":"1732224262727"} 2024-11-21T21:24:22,730 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:24:22,730 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:24:22,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:24:22,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, UNASSIGN}] 2024-11-21T21:24:22,737 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, UNASSIGN 2024-11-21T21:24:22,738 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4b5690ae4b378a5d396cbe15e83372d1, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:22,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, UNASSIGN because future has completed 2024-11-21T21:24:22,741 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:24:22,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:24:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:24:22,898 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:22,898 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:24:22,899 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 4b5690ae4b378a5d396cbe15e83372d1, disabling compactions & flushes 2024-11-21T21:24:22,899 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:22,899 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:22,899 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. after waiting 0 ms 2024-11-21T21:24:22,899 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:22,899 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 4b5690ae4b378a5d396cbe15e83372d1 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-21T21:24:22,903 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/a1bfa9ca1e3f40459787b6d906b11dd5 is 50, key is test_row_2/A:col10/1732224262710/Put/seqid=0 2024-11-21T21:24:22,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741945_1121 (size=7415) 2024-11-21T21:24:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:24:23,309 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/a1bfa9ca1e3f40459787b6d906b11dd5 2024-11-21T21:24:23,323 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/98cdd2d06977487e8330922765e76662 is 50, key is test_row_2/B:col10/1732224262710/Put/seqid=0 2024-11-21T21:24:23,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741946_1122 (size=7415) 2024-11-21T21:24:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:24:23,729 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/98cdd2d06977487e8330922765e76662 2024-11-21T21:24:23,739 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/b1a8c07a87254a88810818a173e3c4f0 is 50, key is test_row_2/C:col10/1732224262710/Put/seqid=0 2024-11-21T21:24:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741947_1123 (size=7415) 2024-11-21T21:24:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:24:24,145 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/b1a8c07a87254a88810818a173e3c4f0 2024-11-21T21:24:24,152 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/A/a1bfa9ca1e3f40459787b6d906b11dd5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/a1bfa9ca1e3f40459787b6d906b11dd5 2024-11-21T21:24:24,159 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/a1bfa9ca1e3f40459787b6d906b11dd5, entries=50, sequenceid=527, filesize=7.2 K 2024-11-21T21:24:24,160 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/B/98cdd2d06977487e8330922765e76662 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/98cdd2d06977487e8330922765e76662 2024-11-21T21:24:24,166 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/98cdd2d06977487e8330922765e76662, entries=50, sequenceid=527, filesize=7.2 K 2024-11-21T21:24:24,167 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/.tmp/C/b1a8c07a87254a88810818a173e3c4f0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/b1a8c07a87254a88810818a173e3c4f0 2024-11-21T21:24:24,172 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/b1a8c07a87254a88810818a173e3c4f0, entries=50, sequenceid=527, filesize=7.2 K 2024-11-21T21:24:24,173 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 4b5690ae4b378a5d396cbe15e83372d1 in 1274ms, sequenceid=527, compaction requested=false 2024-11-21T21:24:24,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/8748bf12e7fe4333bd88aedc7b5808b7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0f3e41bd59154355af4fd1666ada9c81, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df449c51ed1844ef93f6c570e8a9718a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/d6e45ce7ddf64caeb25f7b3eba724828, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e6b76cddeee34be8b09dd644eea8b815, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fe536449a24c4e139f319b661f4b6d0a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/41eb15c1f71743bfa8b4135c8f318c7c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/dc821f4a585c4c92adab0039d013933f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9f5a8c4fa28e42bebd08e6700fc2d243, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f14dd8708164483096139fe4b91765c9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/c18de8bd18d045f98c0823fae9b14ce2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e15c8104ebeb4929b3660a1c4745cf87, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/01f552bc8e99462197040780516d1583, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/17b38e0ff2084831b6ff9e6771f3a4ac, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/62474ba9b6684db2879b9fd9a93e0628, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/990fee8879184ccb8063374824b03e1c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df2f6a23377f4bed86a76385d785a636, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/7a93c60a8c7a4f82903eb910456ad583, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b12feab5f4e748a8b6cefdea8c66f839, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9635f4c640674be8b1d6b15a26485960, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6f70efaa1bcb47b996be0819e7115e3c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b7dd9cce3cf342bcbd1ec81c84851286, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0c93d7b04e6a40b994c52bd2ce139cc8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fa24b52473d240f795c7a9634712e88f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/46d6683fbe75465eb510965aa78e7687, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0d0143160c5946729ee59a3ac65c253e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/bcaa31e20e5845388e64283565942a5e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f1083774c4884d928c641e3d10ad956b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6a621320ffd142cabb19faa9f1c8908e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/37835cd2d6c4474cbd857b0b2a1eaa62, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6e55661ce3084967bc7a4810aee766c4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/2b92096753344697978b912196f9ba5f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/225499324ff44b90b19102086112135d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/607f70f62a2b4a54a14b56948b417ade, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e800aa60d70d4c2a88423adc265c8900] to archive 2024-11-21T21:24:24,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:24:24,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/8748bf12e7fe4333bd88aedc7b5808b7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/8748bf12e7fe4333bd88aedc7b5808b7 2024-11-21T21:24:24,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0f3e41bd59154355af4fd1666ada9c81 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0f3e41bd59154355af4fd1666ada9c81 2024-11-21T21:24:24,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df449c51ed1844ef93f6c570e8a9718a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df449c51ed1844ef93f6c570e8a9718a 2024-11-21T21:24:24,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/d6e45ce7ddf64caeb25f7b3eba724828 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/d6e45ce7ddf64caeb25f7b3eba724828 2024-11-21T21:24:24,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e6b76cddeee34be8b09dd644eea8b815 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e6b76cddeee34be8b09dd644eea8b815 2024-11-21T21:24:24,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fe536449a24c4e139f319b661f4b6d0a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fe536449a24c4e139f319b661f4b6d0a 2024-11-21T21:24:24,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/41eb15c1f71743bfa8b4135c8f318c7c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/41eb15c1f71743bfa8b4135c8f318c7c 2024-11-21T21:24:24,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/dc821f4a585c4c92adab0039d013933f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/dc821f4a585c4c92adab0039d013933f 2024-11-21T21:24:24,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9f5a8c4fa28e42bebd08e6700fc2d243 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9f5a8c4fa28e42bebd08e6700fc2d243 2024-11-21T21:24:24,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f14dd8708164483096139fe4b91765c9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f14dd8708164483096139fe4b91765c9 2024-11-21T21:24:24,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/c18de8bd18d045f98c0823fae9b14ce2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/c18de8bd18d045f98c0823fae9b14ce2 2024-11-21T21:24:24,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e15c8104ebeb4929b3660a1c4745cf87 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e15c8104ebeb4929b3660a1c4745cf87 2024-11-21T21:24:24,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/01f552bc8e99462197040780516d1583 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/01f552bc8e99462197040780516d1583 2024-11-21T21:24:24,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/17b38e0ff2084831b6ff9e6771f3a4ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/17b38e0ff2084831b6ff9e6771f3a4ac 2024-11-21T21:24:24,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/62474ba9b6684db2879b9fd9a93e0628 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/62474ba9b6684db2879b9fd9a93e0628 2024-11-21T21:24:24,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/990fee8879184ccb8063374824b03e1c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/990fee8879184ccb8063374824b03e1c 2024-11-21T21:24:24,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df2f6a23377f4bed86a76385d785a636 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/df2f6a23377f4bed86a76385d785a636 2024-11-21T21:24:24,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/7a93c60a8c7a4f82903eb910456ad583 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/7a93c60a8c7a4f82903eb910456ad583 2024-11-21T21:24:24,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b12feab5f4e748a8b6cefdea8c66f839 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b12feab5f4e748a8b6cefdea8c66f839 2024-11-21T21:24:24,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9635f4c640674be8b1d6b15a26485960 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/9635f4c640674be8b1d6b15a26485960 2024-11-21T21:24:24,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6f70efaa1bcb47b996be0819e7115e3c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6f70efaa1bcb47b996be0819e7115e3c 2024-11-21T21:24:24,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b7dd9cce3cf342bcbd1ec81c84851286 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/b7dd9cce3cf342bcbd1ec81c84851286 2024-11-21T21:24:24,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0c93d7b04e6a40b994c52bd2ce139cc8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0c93d7b04e6a40b994c52bd2ce139cc8 2024-11-21T21:24:24,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fa24b52473d240f795c7a9634712e88f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/fa24b52473d240f795c7a9634712e88f 2024-11-21T21:24:24,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/46d6683fbe75465eb510965aa78e7687 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/46d6683fbe75465eb510965aa78e7687 2024-11-21T21:24:24,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0d0143160c5946729ee59a3ac65c253e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/0d0143160c5946729ee59a3ac65c253e 2024-11-21T21:24:24,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/bcaa31e20e5845388e64283565942a5e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/bcaa31e20e5845388e64283565942a5e 2024-11-21T21:24:24,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f1083774c4884d928c641e3d10ad956b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/f1083774c4884d928c641e3d10ad956b 2024-11-21T21:24:24,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6a621320ffd142cabb19faa9f1c8908e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6a621320ffd142cabb19faa9f1c8908e 2024-11-21T21:24:24,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/37835cd2d6c4474cbd857b0b2a1eaa62 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/37835cd2d6c4474cbd857b0b2a1eaa62 2024-11-21T21:24:24,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6e55661ce3084967bc7a4810aee766c4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/6e55661ce3084967bc7a4810aee766c4 2024-11-21T21:24:24,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/2b92096753344697978b912196f9ba5f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/2b92096753344697978b912196f9ba5f 2024-11-21T21:24:24,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/225499324ff44b90b19102086112135d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/225499324ff44b90b19102086112135d 2024-11-21T21:24:24,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/607f70f62a2b4a54a14b56948b417ade to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/607f70f62a2b4a54a14b56948b417ade 2024-11-21T21:24:24,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e800aa60d70d4c2a88423adc265c8900 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/e800aa60d70d4c2a88423adc265c8900 2024-11-21T21:24:24,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2480f351344f47aaa997080548b24ba6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b12779101462fbda0b3dddeb79d8e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/1ae27e902bea4ac5b04ef4a61c80ecca, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/65d7ced47467447087a3aee4636efbe8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/af4cdcb4161e482f9803785bce649fa8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/20f6233f1e0249a0a5d22d32da0c8671, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/6a405279f29f49a5b9da7fb84105167a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8f81f427183347198bcfaaf6acb76b82, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/daca3cffe045491db84d02406e508309, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c45f635903d4a5cb797a90891f12ac5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f646a7d1e379493ca8d3f2d697805862, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f24adfa1c4ee43b1aa4127e20f12b7b9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/706eea3edf6e49efa608a7a8f2ce3418, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2277d796b9f84c2b86f4c36e6ba0ea21, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c522d4f59574a9e88d6ebeed8fe68f1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/fb8afaa49473478cb1536b98061f271d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/cc6f0faa77cc4078a606914f0a449205, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/5768c9cf2ae84d8583ca5e0312c7ad86, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b3cd3575248b1a9dfdd2ae2fcd0dd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/9099786b16584809b764777af842266e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b74105a54ea9498d8cdf4a3cf2522f4b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c313b720e7cd4a408a015cfb8e746a95, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7304478046d84f529fcfc49f8a73b064, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8d2eec21730043ad8cd38bde074559d2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/e434b37958ac446c992b713e31eeda9d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4b4c5bea9e8f45539b2efc5d992a50ba, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2fee48dfce244fec89e4a643470d5a93, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b6802d67f718442a98f04cf5ff540a9a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4ce2a5cd768342c2b479f59439196969, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/66da125a74e84039be4d43011dea7dcc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/07e1e5582bbd419e95807cff6a3fe5d4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ad5e601bea44488c8a2ab420ead4892d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7be4d6cabb18440da96bd37fd50d0c1c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c3185614559a425bb650d93a8df7a67a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ee625fcfd22140eb86a72e77b79add37] to archive 2024-11-21T21:24:24,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:24:24,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2480f351344f47aaa997080548b24ba6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2480f351344f47aaa997080548b24ba6 2024-11-21T21:24:24,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b12779101462fbda0b3dddeb79d8e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b12779101462fbda0b3dddeb79d8e 2024-11-21T21:24:24,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/1ae27e902bea4ac5b04ef4a61c80ecca to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/1ae27e902bea4ac5b04ef4a61c80ecca 2024-11-21T21:24:24,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/65d7ced47467447087a3aee4636efbe8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/65d7ced47467447087a3aee4636efbe8 2024-11-21T21:24:24,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/af4cdcb4161e482f9803785bce649fa8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/af4cdcb4161e482f9803785bce649fa8 2024-11-21T21:24:24,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/20f6233f1e0249a0a5d22d32da0c8671 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/20f6233f1e0249a0a5d22d32da0c8671 2024-11-21T21:24:24,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/6a405279f29f49a5b9da7fb84105167a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/6a405279f29f49a5b9da7fb84105167a 2024-11-21T21:24:24,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8f81f427183347198bcfaaf6acb76b82 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8f81f427183347198bcfaaf6acb76b82 2024-11-21T21:24:24,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/daca3cffe045491db84d02406e508309 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/daca3cffe045491db84d02406e508309 2024-11-21T21:24:24,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c45f635903d4a5cb797a90891f12ac5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c45f635903d4a5cb797a90891f12ac5 2024-11-21T21:24:24,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f646a7d1e379493ca8d3f2d697805862 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f646a7d1e379493ca8d3f2d697805862 2024-11-21T21:24:24,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f24adfa1c4ee43b1aa4127e20f12b7b9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/f24adfa1c4ee43b1aa4127e20f12b7b9 2024-11-21T21:24:24,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/706eea3edf6e49efa608a7a8f2ce3418 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/706eea3edf6e49efa608a7a8f2ce3418 2024-11-21T21:24:24,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2277d796b9f84c2b86f4c36e6ba0ea21 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2277d796b9f84c2b86f4c36e6ba0ea21 2024-11-21T21:24:24,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c522d4f59574a9e88d6ebeed8fe68f1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/3c522d4f59574a9e88d6ebeed8fe68f1 2024-11-21T21:24:24,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/fb8afaa49473478cb1536b98061f271d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/fb8afaa49473478cb1536b98061f271d 2024-11-21T21:24:24,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/cc6f0faa77cc4078a606914f0a449205 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/cc6f0faa77cc4078a606914f0a449205 2024-11-21T21:24:24,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/5768c9cf2ae84d8583ca5e0312c7ad86 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/5768c9cf2ae84d8583ca5e0312c7ad86 2024-11-21T21:24:24,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b3cd3575248b1a9dfdd2ae2fcd0dd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b96b3cd3575248b1a9dfdd2ae2fcd0dd 2024-11-21T21:24:24,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/9099786b16584809b764777af842266e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/9099786b16584809b764777af842266e 2024-11-21T21:24:24,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b74105a54ea9498d8cdf4a3cf2522f4b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b74105a54ea9498d8cdf4a3cf2522f4b 2024-11-21T21:24:24,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c313b720e7cd4a408a015cfb8e746a95 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c313b720e7cd4a408a015cfb8e746a95 2024-11-21T21:24:24,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7304478046d84f529fcfc49f8a73b064 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7304478046d84f529fcfc49f8a73b064 2024-11-21T21:24:24,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8d2eec21730043ad8cd38bde074559d2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/8d2eec21730043ad8cd38bde074559d2 2024-11-21T21:24:24,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/e434b37958ac446c992b713e31eeda9d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/e434b37958ac446c992b713e31eeda9d 2024-11-21T21:24:24,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4b4c5bea9e8f45539b2efc5d992a50ba to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4b4c5bea9e8f45539b2efc5d992a50ba 2024-11-21T21:24:24,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2fee48dfce244fec89e4a643470d5a93 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/2fee48dfce244fec89e4a643470d5a93 2024-11-21T21:24:24,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b6802d67f718442a98f04cf5ff540a9a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/b6802d67f718442a98f04cf5ff540a9a 2024-11-21T21:24:24,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4ce2a5cd768342c2b479f59439196969 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/4ce2a5cd768342c2b479f59439196969 2024-11-21T21:24:24,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/66da125a74e84039be4d43011dea7dcc to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/66da125a74e84039be4d43011dea7dcc 2024-11-21T21:24:24,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/07e1e5582bbd419e95807cff6a3fe5d4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/07e1e5582bbd419e95807cff6a3fe5d4 2024-11-21T21:24:24,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ad5e601bea44488c8a2ab420ead4892d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ad5e601bea44488c8a2ab420ead4892d 2024-11-21T21:24:24,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7be4d6cabb18440da96bd37fd50d0c1c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/7be4d6cabb18440da96bd37fd50d0c1c 2024-11-21T21:24:24,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c3185614559a425bb650d93a8df7a67a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/c3185614559a425bb650d93a8df7a67a 2024-11-21T21:24:24,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ee625fcfd22140eb86a72e77b79add37 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/ee625fcfd22140eb86a72e77b79add37 2024-11-21T21:24:24,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8662333747ae497bb8c8ef3b76d3e54e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/415d4916fc6849649cfd16ec0f2b6011, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/a485b9942d644a18ba12011913529dbb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/254d1758c9f140cc98f93fb2dc2aed3b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c92ad39476b04bd2a3f4a14b7f09a173, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f2c9efd0bef74fde8a1c165ca4976c28, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c3b2870bd33241e5af08987cbf20f598, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5cf60c31f6504271a2d6d77246b4eeea, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8cb387a1cc824cf188ce148b96e477ae, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c05a1193a7ec48d1afc1f2069efa8f2c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/232b8daafc84475d81c5ec2dac278c6c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bb6361488ad440a7b4e0fe546691b6ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f01e4493868246f5943fa8c5611757a5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/66a0643ef84345808e7fea142ccc18f1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/2533439cf7ee4c298db4e405548527fd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5f381f32e1ad47c6b7e60fc7c4997d5d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf94a1430b714406a919d6ab39775fce, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/facb8d71314a41a698de8b04a7dbf15a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/019e09232806409b98a8f067d96c706e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c2870f385c924c888af2ba2215d17c2e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/39b5cb50c33642f49f4bfa6883d5c2f5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/15a364539f4741dd99465c24b5771191, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/489b4371d6004bbaaf1eccda9e8d1281, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/11892aa01bcf4b7c9c5059edfa5a7ea6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c6921708f68a4ed2ab3ee52bd66cc493, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e7317410afee4ec0a78de8a5b339a68d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/eb9683c2e88449f1b83e35b524fc0168, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bd4e8d567592404b967117bc17069ed0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf3dade3b9aa4a20ab6a33e503e10969, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/d30787d21ebf4e2cb52ec522fb516911, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/dbc1db4358cf43b7a0f8f60fa169f63c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1e2226c38b314ba6890516db339bbe57, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1fa4fde9ad504a12869760b546c96570, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e3285349328d4d97b21846631e471e37] to archive 2024-11-21T21:24:24,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:24:24,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8662333747ae497bb8c8ef3b76d3e54e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8662333747ae497bb8c8ef3b76d3e54e 2024-11-21T21:24:24,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/415d4916fc6849649cfd16ec0f2b6011 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/415d4916fc6849649cfd16ec0f2b6011 2024-11-21T21:24:24,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/a485b9942d644a18ba12011913529dbb to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/a485b9942d644a18ba12011913529dbb 2024-11-21T21:24:24,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/254d1758c9f140cc98f93fb2dc2aed3b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/254d1758c9f140cc98f93fb2dc2aed3b 2024-11-21T21:24:24,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c92ad39476b04bd2a3f4a14b7f09a173 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c92ad39476b04bd2a3f4a14b7f09a173 2024-11-21T21:24:24,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f2c9efd0bef74fde8a1c165ca4976c28 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f2c9efd0bef74fde8a1c165ca4976c28 2024-11-21T21:24:24,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c3b2870bd33241e5af08987cbf20f598 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c3b2870bd33241e5af08987cbf20f598 2024-11-21T21:24:24,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5cf60c31f6504271a2d6d77246b4eeea to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5cf60c31f6504271a2d6d77246b4eeea 2024-11-21T21:24:24,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8cb387a1cc824cf188ce148b96e477ae to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/8cb387a1cc824cf188ce148b96e477ae 2024-11-21T21:24:24,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c05a1193a7ec48d1afc1f2069efa8f2c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c05a1193a7ec48d1afc1f2069efa8f2c 2024-11-21T21:24:24,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/232b8daafc84475d81c5ec2dac278c6c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/232b8daafc84475d81c5ec2dac278c6c 2024-11-21T21:24:24,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bb6361488ad440a7b4e0fe546691b6ff to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bb6361488ad440a7b4e0fe546691b6ff 2024-11-21T21:24:24,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f01e4493868246f5943fa8c5611757a5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/f01e4493868246f5943fa8c5611757a5 2024-11-21T21:24:24,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/66a0643ef84345808e7fea142ccc18f1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/66a0643ef84345808e7fea142ccc18f1 2024-11-21T21:24:24,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/2533439cf7ee4c298db4e405548527fd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/2533439cf7ee4c298db4e405548527fd 2024-11-21T21:24:24,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5f381f32e1ad47c6b7e60fc7c4997d5d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/5f381f32e1ad47c6b7e60fc7c4997d5d 2024-11-21T21:24:24,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf94a1430b714406a919d6ab39775fce to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf94a1430b714406a919d6ab39775fce 2024-11-21T21:24:24,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/facb8d71314a41a698de8b04a7dbf15a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/facb8d71314a41a698de8b04a7dbf15a 2024-11-21T21:24:24,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/019e09232806409b98a8f067d96c706e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/019e09232806409b98a8f067d96c706e 2024-11-21T21:24:24,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c09a9e8c97bd4e3aa7a2c1d63804d2d0 2024-11-21T21:24:24,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c2870f385c924c888af2ba2215d17c2e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c2870f385c924c888af2ba2215d17c2e 2024-11-21T21:24:24,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/39b5cb50c33642f49f4bfa6883d5c2f5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/39b5cb50c33642f49f4bfa6883d5c2f5 2024-11-21T21:24:24,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/15a364539f4741dd99465c24b5771191 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/15a364539f4741dd99465c24b5771191 2024-11-21T21:24:24,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/489b4371d6004bbaaf1eccda9e8d1281 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/489b4371d6004bbaaf1eccda9e8d1281 2024-11-21T21:24:24,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/11892aa01bcf4b7c9c5059edfa5a7ea6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/11892aa01bcf4b7c9c5059edfa5a7ea6 2024-11-21T21:24:24,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c6921708f68a4ed2ab3ee52bd66cc493 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/c6921708f68a4ed2ab3ee52bd66cc493 2024-11-21T21:24:24,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e7317410afee4ec0a78de8a5b339a68d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e7317410afee4ec0a78de8a5b339a68d 2024-11-21T21:24:24,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/eb9683c2e88449f1b83e35b524fc0168 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/eb9683c2e88449f1b83e35b524fc0168 2024-11-21T21:24:24,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bd4e8d567592404b967117bc17069ed0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bd4e8d567592404b967117bc17069ed0 2024-11-21T21:24:24,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf3dade3b9aa4a20ab6a33e503e10969 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/bf3dade3b9aa4a20ab6a33e503e10969 2024-11-21T21:24:24,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/d30787d21ebf4e2cb52ec522fb516911 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/d30787d21ebf4e2cb52ec522fb516911 2024-11-21T21:24:24,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/dbc1db4358cf43b7a0f8f60fa169f63c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/dbc1db4358cf43b7a0f8f60fa169f63c 2024-11-21T21:24:24,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1e2226c38b314ba6890516db339bbe57 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1e2226c38b314ba6890516db339bbe57 2024-11-21T21:24:24,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1fa4fde9ad504a12869760b546c96570 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/1fa4fde9ad504a12869760b546c96570 2024-11-21T21:24:24,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e3285349328d4d97b21846631e471e37 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/e3285349328d4d97b21846631e471e37 2024-11-21T21:24:24,388 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/recovered.edits/530.seqid, newMaxSeqId=530, maxSeqId=1 2024-11-21T21:24:24,390 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. 2024-11-21T21:24:24,390 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 4b5690ae4b378a5d396cbe15e83372d1: Waiting for close lock at 1732224262898Running coprocessor pre-close hooks at 1732224262898Disabling compacts and flushes for region at 1732224262898Disabling writes for close at 1732224262899 (+1 ms)Obtaining lock to block concurrent updates at 1732224262899Preparing flush snapshotting stores in 4b5690ae4b378a5d396cbe15e83372d1 at 1732224262899Finished memstore snapshotting TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732224262899Flushing stores of TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1. at 1732224262900 (+1 ms)Flushing 4b5690ae4b378a5d396cbe15e83372d1/A: creating writer at 1732224262900Flushing 4b5690ae4b378a5d396cbe15e83372d1/A: appending metadata at 1732224262903 (+3 ms)Flushing 4b5690ae4b378a5d396cbe15e83372d1/A: closing flushed file at 1732224262903Flushing 4b5690ae4b378a5d396cbe15e83372d1/B: creating writer at 1732224263318 (+415 ms)Flushing 4b5690ae4b378a5d396cbe15e83372d1/B: appending metadata at 1732224263322 (+4 ms)Flushing 4b5690ae4b378a5d396cbe15e83372d1/B: closing flushed file at 1732224263322Flushing 4b5690ae4b378a5d396cbe15e83372d1/C: creating writer at 1732224263735 (+413 ms)Flushing 4b5690ae4b378a5d396cbe15e83372d1/C: appending metadata at 1732224263738 (+3 ms)Flushing 4b5690ae4b378a5d396cbe15e83372d1/C: closing flushed file at 1732224263739 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e1256b8: reopening flushed file at 1732224264151 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37498351: reopening flushed file at 1732224264159 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@445501a0: reopening flushed file at 1732224264166 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 4b5690ae4b378a5d396cbe15e83372d1 in 1274ms, sequenceid=527, compaction requested=false at 1732224264173 (+7 ms)Writing region close event to WAL at 1732224264384 (+211 ms)Running coprocessor post-close hooks at 1732224264388 (+4 ms)Closed at 1732224264390 (+2 ms) 2024-11-21T21:24:24,393 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:24,394 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4b5690ae4b378a5d396cbe15e83372d1, regionState=CLOSED 2024-11-21T21:24:24,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:24:24,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T21:24:24,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 4b5690ae4b378a5d396cbe15e83372d1, server=849dee24a242,34123,1732224229084 in 1.6560 sec 2024-11-21T21:24:24,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-21T21:24:24,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b5690ae4b378a5d396cbe15e83372d1, UNASSIGN in 1.6630 sec 2024-11-21T21:24:24,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T21:24:24,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6690 sec 2024-11-21T21:24:24,405 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224264405"}]},"ts":"1732224264405"} 2024-11-21T21:24:24,406 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:24:24,406 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:24:24,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6850 sec 2024-11-21T21:24:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:24:24,870 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:24,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:24:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:24,880 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T21:24:24,882 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:24,886 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:24,889 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/recovered.edits] 2024-11-21T21:24:24,892 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/777ebe36e1754b6aa0502feaaa1329ba to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/777ebe36e1754b6aa0502feaaa1329ba 2024-11-21T21:24:24,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/a1bfa9ca1e3f40459787b6d906b11dd5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/A/a1bfa9ca1e3f40459787b6d906b11dd5 2024-11-21T21:24:24,896 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/303c6c75e2694007adf33233b1f9cb4f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/303c6c75e2694007adf33233b1f9cb4f 2024-11-21T21:24:24,898 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/98cdd2d06977487e8330922765e76662 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/B/98cdd2d06977487e8330922765e76662 2024-11-21T21:24:24,900 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/620af1b948014decbc7a392733e1c52c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/620af1b948014decbc7a392733e1c52c 2024-11-21T21:24:24,901 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/b1a8c07a87254a88810818a173e3c4f0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/C/b1a8c07a87254a88810818a173e3c4f0 2024-11-21T21:24:24,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/recovered.edits/530.seqid to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1/recovered.edits/530.seqid 2024-11-21T21:24:24,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/4b5690ae4b378a5d396cbe15e83372d1 2024-11-21T21:24:24,905 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:24:24,910 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:24,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T21:24:24,920 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:24:24,948 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:24:24,950 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:24,950 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:24:24,950 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732224264950"}]},"ts":"9223372036854775807"} 2024-11-21T21:24:24,954 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:24:24,954 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4b5690ae4b378a5d396cbe15e83372d1, NAME => 'TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:24:24,955 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:24:24,955 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732224264955"}]},"ts":"9223372036854775807"} 2024-11-21T21:24:24,958 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:24:24,958 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:24,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-21T21:24:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T21:24:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:24:24,990 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:25,001 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;849dee24a242:34123-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1390536951_22 at /127.0.0.1:50644 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 526), ProcessCount=11 (was 11), AvailableMemoryMB=12143 (was 12946) 2024-11-21T21:24:25,010 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=12142 2024-11-21T21:24:25,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:24:25,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:24:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:25,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:24:25,014 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:25,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-21T21:24:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T21:24:25,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:24:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741948_1124 (size=959) 2024-11-21T21:24:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T21:24:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T21:24:25,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:24:25,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741949_1125 (size=53) 2024-11-21T21:24:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T21:24:25,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:24:25,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cd5da8e0c287760dca4b3d875dd531a4, disabling compactions & flushes 2024-11-21T21:24:25,836 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:25,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:25,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. after waiting 0 ms 2024-11-21T21:24:25,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:25,836 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:25,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cd5da8e0c287760dca4b3d875dd531a4: Waiting for close lock at 1732224265836Disabling compacts and flushes for region at 1732224265836Disabling writes for close at 1732224265836Writing region close event to WAL at 1732224265836Closed at 1732224265836 2024-11-21T21:24:25,838 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:24:25,838 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732224265838"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732224265838"}]},"ts":"1732224265838"} 2024-11-21T21:24:25,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:24:25,841 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:24:25,841 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224265841"}]},"ts":"1732224265841"} 2024-11-21T21:24:25,843 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:24:25,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, ASSIGN}] 2024-11-21T21:24:25,845 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, ASSIGN 2024-11-21T21:24:25,845 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, ASSIGN; state=OFFLINE, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=false 2024-11-21T21:24:25,996 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:25,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, ASSIGN because future has completed 2024-11-21T21:24:25,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:24:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T21:24:26,159 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:26,160 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:24:26,160 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,160 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:24:26,161 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,161 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,163 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,167 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd5da8e0c287760dca4b3d875dd531a4 columnFamilyName A 2024-11-21T21:24:26,168 DEBUG [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:26,169 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(327): Store=cd5da8e0c287760dca4b3d875dd531a4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:26,169 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,172 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd5da8e0c287760dca4b3d875dd531a4 columnFamilyName B 2024-11-21T21:24:26,172 DEBUG [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:26,173 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(327): Store=cd5da8e0c287760dca4b3d875dd531a4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:26,173 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,174 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd5da8e0c287760dca4b3d875dd531a4 columnFamilyName C 2024-11-21T21:24:26,174 DEBUG [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:26,174 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(327): Store=cd5da8e0c287760dca4b3d875dd531a4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:26,175 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,175 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,176 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,177 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,177 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,177 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:24:26,178 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,180 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:24:26,180 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened cd5da8e0c287760dca4b3d875dd531a4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68065278, jitterRate=0.014251679182052612}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:24:26,180 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:26,181 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for cd5da8e0c287760dca4b3d875dd531a4: Running coprocessor pre-open hook at 1732224266161Writing region info on filesystem at 1732224266161Initializing all the Stores at 1732224266162 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224266163 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224266163Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224266163Cleaning up temporary data from old regions at 1732224266177 (+14 ms)Running coprocessor post-open hooks at 1732224266180 (+3 ms)Region opened successfully at 1732224266181 (+1 ms) 2024-11-21T21:24:26,182 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., pid=32, masterSystemTime=1732224266151 2024-11-21T21:24:26,183 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:26,183 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:26,184 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:26,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:24:26,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-21T21:24:26,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 in 188 msec 2024-11-21T21:24:26,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-21T21:24:26,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, ASSIGN in 345 msec 2024-11-21T21:24:26,191 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:24:26,191 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224266191"}]},"ts":"1732224266191"} 2024-11-21T21:24:26,193 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:24:26,194 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:24:26,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-21T21:24:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T21:24:27,160 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:27,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:27,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:27,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:27,165 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:27,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717368f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:27,169 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf8b3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:27,170 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:27,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:27,175 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:24:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,35047,1732224227985 2024-11-21T21:24:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ad97762 2024-11-21T21:24:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:24:27,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:24:27,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:24:27,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:24:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:27,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741950_1126 (size=995) 2024-11-21T21:24:27,605 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T21:24:27,606 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T21:24:27,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:24:27,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, REOPEN/MOVE}] 2024-11-21T21:24:27,618 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, REOPEN/MOVE 2024-11-21T21:24:27,618 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:27,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, REOPEN/MOVE because future has completed 2024-11-21T21:24:27,621 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:24:27,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:24:27,774 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:27,774 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:24:27,774 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing cd5da8e0c287760dca4b3d875dd531a4, disabling compactions & flushes 2024-11-21T21:24:27,775 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:27,775 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:27,775 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. after waiting 0 ms 2024-11-21T21:24:27,775 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:27,779 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T21:24:27,780 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:27,780 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for cd5da8e0c287760dca4b3d875dd531a4: Waiting for close lock at 1732224267774Running coprocessor pre-close hooks at 1732224267774Disabling compacts and flushes for region at 1732224267774Disabling writes for close at 1732224267775 (+1 ms)Writing region close event to WAL at 1732224267775Running coprocessor post-close hooks at 1732224267780 (+5 ms)Closed at 1732224267780 2024-11-21T21:24:27,781 WARN [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: cd5da8e0c287760dca4b3d875dd531a4 to self. 2024-11-21T21:24:27,783 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:27,784 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=CLOSED 2024-11-21T21:24:27,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:24:27,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T21:24:27,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 in 168 msec 2024-11-21T21:24:27,792 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, REOPEN/MOVE; state=CLOSED, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=true 2024-11-21T21:24:27,942 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:27,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, REOPEN/MOVE because future has completed 2024-11-21T21:24:27,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:24:28,101 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,101 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:24:28,102 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,102 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:24:28,102 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,102 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,106 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,112 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd5da8e0c287760dca4b3d875dd531a4 columnFamilyName A 2024-11-21T21:24:28,114 DEBUG [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:28,114 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(327): Store=cd5da8e0c287760dca4b3d875dd531a4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:28,115 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,116 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd5da8e0c287760dca4b3d875dd531a4 columnFamilyName B 2024-11-21T21:24:28,116 DEBUG [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:28,117 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(327): Store=cd5da8e0c287760dca4b3d875dd531a4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:28,117 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,118 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd5da8e0c287760dca4b3d875dd531a4 columnFamilyName C 2024-11-21T21:24:28,118 DEBUG [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:28,119 INFO [StoreOpener-cd5da8e0c287760dca4b3d875dd531a4-1 {}] regionserver.HStore(327): Store=cd5da8e0c287760dca4b3d875dd531a4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:28,119 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,120 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,121 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,122 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,122 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,123 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:24:28,124 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,125 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened cd5da8e0c287760dca4b3d875dd531a4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58951001, jitterRate=-0.12156163156032562}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:24:28,125 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,127 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for cd5da8e0c287760dca4b3d875dd531a4: Running coprocessor pre-open hook at 1732224268102Writing region info on filesystem at 1732224268102Initializing all the Stores at 1732224268103 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224268103Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224268103Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224268103Cleaning up temporary data from old regions at 1732224268122 (+19 ms)Running coprocessor post-open hooks at 1732224268125 (+3 ms)Region opened successfully at 1732224268126 (+1 ms) 2024-11-21T21:24:28,127 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., pid=37, masterSystemTime=1732224268098 2024-11-21T21:24:28,129 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,129 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,130 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=OPEN, openSeqNum=5, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:24:28,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-21T21:24:28,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 in 188 msec 2024-11-21T21:24:28,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-21T21:24:28,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, REOPEN/MOVE in 517 msec 2024-11-21T21:24:28,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T21:24:28,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 529 msec 2024-11-21T21:24:28,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-11-21T21:24:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T21:24:28,146 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:28,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,148 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bdf31b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,150 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0451e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,153 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2947886c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,155 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e65913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a42d08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b809a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f381c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,164 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f06496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,167 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1259d01a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,169 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25357054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,172 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667916b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,174 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b118760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e09993, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,178 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46bfc8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,181 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ba5a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,182 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679bff6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:24:28,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:24:28,185 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:24:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:24:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:24:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ca9cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:24:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:24:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:28,187 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:24:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:24:28,191 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,193 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-21T21:24:28,194 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,195 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:28,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T21:24:28,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,220 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,232 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,244 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,249 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:24:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,262 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,276 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:24:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:24:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:24:28,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:24:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:24:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T21:24:28,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224328307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224328309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224328311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224328313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224328313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210f4a18f4a7fe4f50b9a39af45fcaa3d4_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224268228/Put/seqid=0 2024-11-21T21:24:28,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T21:24:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741951_1127 (size=19474) 2024-11-21T21:24:28,378 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:28,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210f4a18f4a7fe4f50b9a39af45fcaa3d4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210f4a18f4a7fe4f50b9a39af45fcaa3d4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/eff7e45c122f4659a9b8a75ee1f8f559, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:28,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/eff7e45c122f4659a9b8a75ee1f8f559 is 175, key is test_row_0/A:col10/1732224268228/Put/seqid=0 2024-11-21T21:24:28,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224328420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741952_1128 (size=56733) 2024-11-21T21:24:28,428 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/eff7e45c122f4659a9b8a75ee1f8f559 2024-11-21T21:24:28,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224328430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224328430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224328430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224328430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/450aa89574c3469fbe39e0f079664a0d is 50, key is test_row_0/B:col10/1732224268228/Put/seqid=0 2024-11-21T21:24:28,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741953_1129 (size=12001) 2024-11-21T21:24:28,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/450aa89574c3469fbe39e0f079664a0d 2024-11-21T21:24:28,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T21:24:28,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T21:24:28,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/04f1b8dd73fb4eb1bb24eff3fa84219b is 50, key is test_row_0/C:col10/1732224268228/Put/seqid=0 2024-11-21T21:24:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741954_1130 (size=12001) 2024-11-21T21:24:28,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/04f1b8dd73fb4eb1bb24eff3fa84219b 2024-11-21T21:24:28,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/eff7e45c122f4659a9b8a75ee1f8f559 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/eff7e45c122f4659a9b8a75ee1f8f559 2024-11-21T21:24:28,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/eff7e45c122f4659a9b8a75ee1f8f559, entries=300, sequenceid=15, filesize=55.4 K 2024-11-21T21:24:28,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/450aa89574c3469fbe39e0f079664a0d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/450aa89574c3469fbe39e0f079664a0d 2024-11-21T21:24:28,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/450aa89574c3469fbe39e0f079664a0d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T21:24:28,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/04f1b8dd73fb4eb1bb24eff3fa84219b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/04f1b8dd73fb4eb1bb24eff3fa84219b 2024-11-21T21:24:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/04f1b8dd73fb4eb1bb24eff3fa84219b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T21:24:28,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for cd5da8e0c287760dca4b3d875dd531a4 in 352ms, sequenceid=15, compaction requested=false 2024-11-21T21:24:28,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,642 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:28,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T21:24:28,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c929f60d9f74421abeac160ae4d9501d_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224268310/Put/seqid=0 2024-11-21T21:24:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224328649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224328652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224328652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T21:24:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741955_1131 (size=14594) 2024-11-21T21:24:28,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,673 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224328659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224328662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,680 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c929f60d9f74421abeac160ae4d9501d_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c929f60d9f74421abeac160ae4d9501d_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:28,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/1e758a120ec24bd1b3a5b0161d887eb4, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:28,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/1e758a120ec24bd1b3a5b0161d887eb4 is 175, key is test_row_0/A:col10/1732224268310/Put/seqid=0 2024-11-21T21:24:28,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741956_1132 (size=39549) 2024-11-21T21:24:28,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/1e758a120ec24bd1b3a5b0161d887eb4 2024-11-21T21:24:28,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/fbd99840c4aa42e0986c0fbae1d74fa9 is 50, key is test_row_0/B:col10/1732224268310/Put/seqid=0 2024-11-21T21:24:28,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741957_1133 (size=12001) 2024-11-21T21:24:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224328770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224328770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224328771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224328780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224328780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T21:24:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T21:24:28,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T21:24:28,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:28,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:28,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224328981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224328981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224328981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224328990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224328992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/fbd99840c4aa42e0986c0fbae1d74fa9 2024-11-21T21:24:29,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T21:24:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:29,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:29,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/b0bbed8b536f451e9fc5ced28aa6917b is 50, key is test_row_0/C:col10/1732224268310/Put/seqid=0 2024-11-21T21:24:29,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741958_1134 (size=12001) 2024-11-21T21:24:29,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/b0bbed8b536f451e9fc5ced28aa6917b 2024-11-21T21:24:29,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/1e758a120ec24bd1b3a5b0161d887eb4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e758a120ec24bd1b3a5b0161d887eb4 2024-11-21T21:24:29,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e758a120ec24bd1b3a5b0161d887eb4, entries=200, sequenceid=42, filesize=38.6 K 2024-11-21T21:24:29,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/fbd99840c4aa42e0986c0fbae1d74fa9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fbd99840c4aa42e0986c0fbae1d74fa9 2024-11-21T21:24:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fbd99840c4aa42e0986c0fbae1d74fa9, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T21:24:29,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/b0bbed8b536f451e9fc5ced28aa6917b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b0bbed8b536f451e9fc5ced28aa6917b 2024-11-21T21:24:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b0bbed8b536f451e9fc5ced28aa6917b, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T21:24:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for cd5da8e0c287760dca4b3d875dd531a4 in 588ms, sequenceid=42, compaction requested=false 2024-11-21T21:24:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T21:24:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:29,288 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T21:24:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214d3f4658cd1c4331969d081b87fa02c4_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224268658/Put/seqid=0 2024-11-21T21:24:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T21:24:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741959_1135 (size=9714) 2024-11-21T21:24:29,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,372 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214d3f4658cd1c4331969d081b87fa02c4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214d3f4658cd1c4331969d081b87fa02c4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/464b9a6a313545d5bfd9f246b59d84ff, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/464b9a6a313545d5bfd9f246b59d84ff is 175, key is test_row_0/A:col10/1732224268658/Put/seqid=0 2024-11-21T21:24:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741960_1136 (size=22361) 2024-11-21T21:24:29,417 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/464b9a6a313545d5bfd9f246b59d84ff 2024-11-21T21:24:29,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a898a3db25114b60b0c687c47fa0f7b2 is 50, key is test_row_0/B:col10/1732224268658/Put/seqid=0 2024-11-21T21:24:29,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224329437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224329441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224329444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224329455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224329457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741961_1137 (size=9657) 2024-11-21T21:24:29,509 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a898a3db25114b60b0c687c47fa0f7b2 2024-11-21T21:24:29,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/0d77f11b57f24653a717093dd71b44ce is 50, key is test_row_0/C:col10/1732224268658/Put/seqid=0 2024-11-21T21:24:29,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741962_1138 (size=9657) 2024-11-21T21:24:29,547 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/0d77f11b57f24653a717093dd71b44ce 2024-11-21T21:24:29,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/464b9a6a313545d5bfd9f246b59d84ff as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/464b9a6a313545d5bfd9f246b59d84ff 2024-11-21T21:24:29,565 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/464b9a6a313545d5bfd9f246b59d84ff, entries=100, sequenceid=51, filesize=21.8 K 2024-11-21T21:24:29,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a898a3db25114b60b0c687c47fa0f7b2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a898a3db25114b60b0c687c47fa0f7b2 2024-11-21T21:24:29,578 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a898a3db25114b60b0c687c47fa0f7b2, entries=100, sequenceid=51, filesize=9.4 K 2024-11-21T21:24:29,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/0d77f11b57f24653a717093dd71b44ce as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0d77f11b57f24653a717093dd71b44ce 2024-11-21T21:24:29,586 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0d77f11b57f24653a717093dd71b44ce, entries=100, sequenceid=51, filesize=9.4 K 2024-11-21T21:24:29,593 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for cd5da8e0c287760dca4b3d875dd531a4 in 300ms, sequenceid=51, compaction requested=true 2024-11-21T21:24:29,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:29,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:29,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-21T21:24:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-21T21:24:29,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-21T21:24:29,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-21T21:24:29,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4050 sec 2024-11-21T21:24:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:29,601 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:29,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-21T21:24:29,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224329602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224329602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224329608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224329610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224329610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f4fd9a30ccd4e6886c04d7263c68f8f_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224269593/Put/seqid=0 2024-11-21T21:24:29,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741963_1139 (size=17034) 2024-11-21T21:24:29,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:29,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f4fd9a30ccd4e6886c04d7263c68f8f_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f4fd9a30ccd4e6886c04d7263c68f8f_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:29,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/877ecdee0790447d844a87e097338941, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:29,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/877ecdee0790447d844a87e097338941 is 175, key is test_row_0/A:col10/1732224269593/Put/seqid=0 2024-11-21T21:24:29,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741964_1140 (size=48139) 2024-11-21T21:24:29,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224329720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224329720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224329720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224329730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224329730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224329930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224329930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224329931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,952 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:24:29,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224329950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224329951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/877ecdee0790447d844a87e097338941 2024-11-21T21:24:30,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a3b54c5747594abb84561e2fec1c23fe is 50, key is test_row_0/B:col10/1732224269593/Put/seqid=0 2024-11-21T21:24:30,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741965_1141 (size=12001) 2024-11-21T21:24:30,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a3b54c5747594abb84561e2fec1c23fe 2024-11-21T21:24:30,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/30185d0c344b4b9595b3c0d15d12152f is 50, key is test_row_0/C:col10/1732224269593/Put/seqid=0 2024-11-21T21:24:30,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741966_1142 (size=12001) 2024-11-21T21:24:30,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/30185d0c344b4b9595b3c0d15d12152f 2024-11-21T21:24:30,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/877ecdee0790447d844a87e097338941 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/877ecdee0790447d844a87e097338941 2024-11-21T21:24:30,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224330241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224330241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224330241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/877ecdee0790447d844a87e097338941, entries=250, sequenceid=82, filesize=47.0 K 2024-11-21T21:24:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a3b54c5747594abb84561e2fec1c23fe as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a3b54c5747594abb84561e2fec1c23fe 2024-11-21T21:24:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224330260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224330260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a3b54c5747594abb84561e2fec1c23fe, entries=150, sequenceid=82, filesize=11.7 K 2024-11-21T21:24:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/30185d0c344b4b9595b3c0d15d12152f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/30185d0c344b4b9595b3c0d15d12152f 2024-11-21T21:24:30,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/30185d0c344b4b9595b3c0d15d12152f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-21T21:24:30,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for cd5da8e0c287760dca4b3d875dd531a4 in 672ms, sequenceid=82, compaction requested=true 2024-11-21T21:24:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:30,273 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:30,273 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:30,276 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:30,276 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:30,276 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,276 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/eff7e45c122f4659a9b8a75ee1f8f559, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e758a120ec24bd1b3a5b0161d887eb4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/464b9a6a313545d5bfd9f246b59d84ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/877ecdee0790447d844a87e097338941] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=162.9 K 2024-11-21T21:24:30,276 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,277 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/eff7e45c122f4659a9b8a75ee1f8f559, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e758a120ec24bd1b3a5b0161d887eb4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/464b9a6a313545d5bfd9f246b59d84ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/877ecdee0790447d844a87e097338941] 2024-11-21T21:24:30,277 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:30,277 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:30,278 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,278 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/450aa89574c3469fbe39e0f079664a0d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fbd99840c4aa42e0986c0fbae1d74fa9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a898a3db25114b60b0c687c47fa0f7b2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a3b54c5747594abb84561e2fec1c23fe] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=44.6 K 2024-11-21T21:24:30,278 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting eff7e45c122f4659a9b8a75ee1f8f559, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732224268214 2024-11-21T21:24:30,278 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 450aa89574c3469fbe39e0f079664a0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732224268228 2024-11-21T21:24:30,281 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e758a120ec24bd1b3a5b0161d887eb4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732224268304 2024-11-21T21:24:30,281 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd99840c4aa42e0986c0fbae1d74fa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732224268304 2024-11-21T21:24:30,282 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a898a3db25114b60b0c687c47fa0f7b2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224268647 2024-11-21T21:24:30,283 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464b9a6a313545d5bfd9f246b59d84ff, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224268647 2024-11-21T21:24:30,284 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b54c5747594abb84561e2fec1c23fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732224269440 2024-11-21T21:24:30,285 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 877ecdee0790447d844a87e097338941, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732224269440 2024-11-21T21:24:30,307 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:30,307 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a17cc3c19de54469b0edcd4a5997a284 is 50, key is test_row_0/B:col10/1732224269593/Put/seqid=0 2024-11-21T21:24:30,337 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T21:24:30,350 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:30,352 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T21:24:30,355 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:30,357 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:30,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:24:30,361 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411215d2f24f3aba648cf976033d585e56ddf_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:30,365 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411215d2f24f3aba648cf976033d585e56ddf_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:30,366 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215d2f24f3aba648cf976033d585e56ddf_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:30,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741967_1143 (size=12139) 2024-11-21T21:24:30,393 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/a17cc3c19de54469b0edcd4a5997a284 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a17cc3c19de54469b0edcd4a5997a284 2024-11-21T21:24:30,402 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into a17cc3c19de54469b0edcd4a5997a284(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:30,402 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:30,402 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=12, startTime=1732224270273; duration=0sec 2024-11-21T21:24:30,402 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:30,402 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:30,402 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:30,406 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:30,406 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:30,406 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,406 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/04f1b8dd73fb4eb1bb24eff3fa84219b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b0bbed8b536f451e9fc5ced28aa6917b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0d77f11b57f24653a717093dd71b44ce, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/30185d0c344b4b9595b3c0d15d12152f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=44.6 K 2024-11-21T21:24:30,406 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 04f1b8dd73fb4eb1bb24eff3fa84219b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732224268228 2024-11-21T21:24:30,407 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bbed8b536f451e9fc5ced28aa6917b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732224268304 2024-11-21T21:24:30,407 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d77f11b57f24653a717093dd71b44ce, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224268647 2024-11-21T21:24:30,407 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 30185d0c344b4b9595b3c0d15d12152f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732224269440 2024-11-21T21:24:30,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741968_1144 (size=4469) 2024-11-21T21:24:30,421 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#124 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:30,421 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/e096796f9bf44eb8b9d63973466b5005 is 175, key is test_row_0/A:col10/1732224269593/Put/seqid=0 2024-11-21T21:24:30,435 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#125 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:30,435 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/4c0d15c6b09c4fe4a0226bf342e9a458 is 50, key is test_row_0/C:col10/1732224269593/Put/seqid=0 2024-11-21T21:24:30,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741969_1145 (size=31093) 2024-11-21T21:24:30,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741970_1146 (size=12139) 2024-11-21T21:24:30,469 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/4c0d15c6b09c4fe4a0226bf342e9a458 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/4c0d15c6b09c4fe4a0226bf342e9a458 2024-11-21T21:24:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:24:30,480 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into 4c0d15c6b09c4fe4a0226bf342e9a458(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:30,480 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:30,480 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=12, startTime=1732224270273; duration=0sec 2024-11-21T21:24:30,481 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:30,481 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:30,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:24:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,511 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-21T21:24:30,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212ac91afae8394d69ace03d6e2fc74ded_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224269608/Put/seqid=0 2024-11-21T21:24:30,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741971_1147 (size=12154) 2024-11-21T21:24:30,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,559 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212ac91afae8394d69ace03d6e2fc74ded_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ac91afae8394d69ace03d6e2fc74ded_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:30,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/accf3e4adbb3409d8b20b3fcf40751b8, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:30,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/accf3e4adbb3409d8b20b3fcf40751b8 is 175, key is test_row_0/A:col10/1732224269608/Put/seqid=0 2024-11-21T21:24:30,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741972_1148 (size=30955) 2024-11-21T21:24:30,583 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/accf3e4adbb3409d8b20b3fcf40751b8 2024-11-21T21:24:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/6ad341fe5a3b4e468cbafe4fd5efc3a1 is 50, key is test_row_0/B:col10/1732224269608/Put/seqid=0 2024-11-21T21:24:30,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741973_1149 (size=12001) 2024-11-21T21:24:30,606 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/6ad341fe5a3b4e468cbafe4fd5efc3a1 2024-11-21T21:24:30,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/3c5147b7afd44e53a44c0666c768f04b is 50, key is test_row_0/C:col10/1732224269608/Put/seqid=0 2024-11-21T21:24:30,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741974_1150 (size=12001) 2024-11-21T21:24:30,632 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/3c5147b7afd44e53a44c0666c768f04b 2024-11-21T21:24:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/accf3e4adbb3409d8b20b3fcf40751b8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/accf3e4adbb3409d8b20b3fcf40751b8 2024-11-21T21:24:30,648 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/accf3e4adbb3409d8b20b3fcf40751b8, entries=150, sequenceid=90, filesize=30.2 K 2024-11-21T21:24:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/6ad341fe5a3b4e468cbafe4fd5efc3a1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6ad341fe5a3b4e468cbafe4fd5efc3a1 2024-11-21T21:24:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,656 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6ad341fe5a3b4e468cbafe4fd5efc3a1, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T21:24:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/3c5147b7afd44e53a44c0666c768f04b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3c5147b7afd44e53a44c0666c768f04b 2024-11-21T21:24:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,668 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3c5147b7afd44e53a44c0666c768f04b, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T21:24:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,670 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for cd5da8e0c287760dca4b3d875dd531a4 in 159ms, sequenceid=90, compaction requested=false 2024-11-21T21:24:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T21:24:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T21:24:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T21:24:30,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 315 msec 2024-11-21T21:24:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 324 msec 2024-11-21T21:24:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:24:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,681 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T21:24:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,688 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:24:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,690 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:24:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:30,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:30,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:24:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f7dada38c2284df7a9e155e5f933e8cb_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224270804/Put/seqid=0 2024-11-21T21:24:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,864 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/e096796f9bf44eb8b9d63973466b5005 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e096796f9bf44eb8b9d63973466b5005 2024-11-21T21:24:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224330862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224330866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224330868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,879 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into e096796f9bf44eb8b9d63973466b5005(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:30,879 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:30,879 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=12, startTime=1732224270273; duration=0sec 2024-11-21T21:24:30,879 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:30,879 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224330869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224330871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741976_1152 (size=24358) 2024-11-21T21:24:30,899 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:30,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f7dada38c2284df7a9e155e5f933e8cb_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f7dada38c2284df7a9e155e5f933e8cb_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:30,908 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/62f56a81f13549a0bed3b1fa1be0d4d5, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:30,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/62f56a81f13549a0bed3b1fa1be0d4d5 is 175, key is test_row_0/A:col10/1732224270804/Put/seqid=0 2024-11-21T21:24:30,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741975_1151 (size=73995) 2024-11-21T21:24:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224330981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224330981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224330982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224330990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224330990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:30,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:30,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:30,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:30,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:30,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:24:31,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224331190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224331190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224331190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224331200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224331200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:31,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,314 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/62f56a81f13549a0bed3b1fa1be0d4d5 2024-11-21T21:24:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:24:31,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/963eb19968d047e3a932f510fad20feb is 50, key is test_row_0/B:col10/1732224270804/Put/seqid=0 2024-11-21T21:24:31,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741977_1153 (size=12001) 2024-11-21T21:24:31,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:31,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224331500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224331500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224331500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224331510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224331520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:31,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/963eb19968d047e3a932f510fad20feb 2024-11-21T21:24:31,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/49267ce4b4d74876baab7259b5e0ce29 is 50, key is test_row_0/C:col10/1732224270804/Put/seqid=0 2024-11-21T21:24:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741978_1154 (size=12001) 2024-11-21T21:24:31,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/49267ce4b4d74876baab7259b5e0ce29 2024-11-21T21:24:31,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:31,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/62f56a81f13549a0bed3b1fa1be0d4d5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/62f56a81f13549a0bed3b1fa1be0d4d5 2024-11-21T21:24:31,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/62f56a81f13549a0bed3b1fa1be0d4d5, entries=400, sequenceid=101, filesize=72.3 K 2024-11-21T21:24:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/963eb19968d047e3a932f510fad20feb as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/963eb19968d047e3a932f510fad20feb 2024-11-21T21:24:31,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/963eb19968d047e3a932f510fad20feb, entries=150, sequenceid=101, filesize=11.7 K 2024-11-21T21:24:31,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/49267ce4b4d74876baab7259b5e0ce29 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/49267ce4b4d74876baab7259b5e0ce29 2024-11-21T21:24:31,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/49267ce4b4d74876baab7259b5e0ce29, entries=150, sequenceid=101, filesize=11.7 K 2024-11-21T21:24:31,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cd5da8e0c287760dca4b3d875dd531a4 in 1015ms, sequenceid=101, compaction requested=true 2024-11-21T21:24:31,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:31,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:31,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:31,821 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:31,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:31,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:31,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:31,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:31,821 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:31,823 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:31,823 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:31,823 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,823 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a17cc3c19de54469b0edcd4a5997a284, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6ad341fe5a3b4e468cbafe4fd5efc3a1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/963eb19968d047e3a932f510fad20feb] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=35.3 K 2024-11-21T21:24:31,824 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:31,824 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:31,824 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,824 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e096796f9bf44eb8b9d63973466b5005, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/accf3e4adbb3409d8b20b3fcf40751b8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/62f56a81f13549a0bed3b1fa1be0d4d5] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=132.9 K 2024-11-21T21:24:31,824 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,824 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e096796f9bf44eb8b9d63973466b5005, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/accf3e4adbb3409d8b20b3fcf40751b8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/62f56a81f13549a0bed3b1fa1be0d4d5] 2024-11-21T21:24:31,825 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a17cc3c19de54469b0edcd4a5997a284, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732224269440 2024-11-21T21:24:31,825 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e096796f9bf44eb8b9d63973466b5005, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732224269440 2024-11-21T21:24:31,825 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad341fe5a3b4e468cbafe4fd5efc3a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732224269602 2024-11-21T21:24:31,826 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting accf3e4adbb3409d8b20b3fcf40751b8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732224269602 2024-11-21T21:24:31,826 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 963eb19968d047e3a932f510fad20feb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732224270784 2024-11-21T21:24:31,826 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f56a81f13549a0bed3b1fa1be0d4d5, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732224270783 2024-11-21T21:24:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:24:31,841 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:31,847 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:31,847 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/f8c96d84f24f4d54941ecdbcde912cea is 50, key is test_row_0/B:col10/1732224270804/Put/seqid=0 2024-11-21T21:24:31,852 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112128c8e7f04653444681b0c4f5c50aec9f_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:31,856 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112128c8e7f04653444681b0c4f5c50aec9f_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:31,856 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112128c8e7f04653444681b0c4f5c50aec9f_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:31,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741979_1155 (size=12241) 2024-11-21T21:24:31,872 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/f8c96d84f24f4d54941ecdbcde912cea as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f8c96d84f24f4d54941ecdbcde912cea 2024-11-21T21:24:31,877 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into f8c96d84f24f4d54941ecdbcde912cea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:31,877 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:31,877 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=13, startTime=1732224271821; duration=0sec 2024-11-21T21:24:31,878 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:31,878 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:31,878 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:31,882 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:31,882 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:31,883 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,883 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/4c0d15c6b09c4fe4a0226bf342e9a458, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3c5147b7afd44e53a44c0666c768f04b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/49267ce4b4d74876baab7259b5e0ce29] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=35.3 K 2024-11-21T21:24:31,883 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0d15c6b09c4fe4a0226bf342e9a458, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732224269440 2024-11-21T21:24:31,884 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5147b7afd44e53a44c0666c768f04b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732224269602 2024-11-21T21:24:31,884 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 49267ce4b4d74876baab7259b5e0ce29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732224270784 2024-11-21T21:24:31,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741980_1156 (size=4469) 2024-11-21T21:24:31,890 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#132 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:31,890 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/0d9095df52d74a0ab55429ed167b7c14 is 175, key is test_row_0/A:col10/1732224270804/Put/seqid=0 2024-11-21T21:24:31,903 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:31,904 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/e96ca9bd774f4817967e86796b99388f is 50, key is test_row_0/C:col10/1732224270804/Put/seqid=0 2024-11-21T21:24:31,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741981_1157 (size=31195) 2024-11-21T21:24:31,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:24:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:31,924 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:24:31,937 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/0d9095df52d74a0ab55429ed167b7c14 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0d9095df52d74a0ab55429ed167b7c14 2024-11-21T21:24:31,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741982_1158 (size=12241) 2024-11-21T21:24:31,957 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into 0d9095df52d74a0ab55429ed167b7c14(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:31,958 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:31,958 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=13, startTime=1732224271821; duration=0sec 2024-11-21T21:24:31,958 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:31,958 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121269e37c3c7624dde87b5d5b165a64317_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224270851/Put/seqid=0 2024-11-21T21:24:32,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741983_1159 (size=12154) 2024-11-21T21:24:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:32,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:32,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224332030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224332030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224332032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224332035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224332036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224332140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224332141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224332141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224332160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224332349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224332349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,361 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/e96ca9bd774f4817967e86796b99388f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/e96ca9bd774f4817967e86796b99388f 2024-11-21T21:24:32,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224332360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,374 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into e96ca9bd774f4817967e86796b99388f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:32,374 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:32,374 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=13, startTime=1732224271821; duration=0sec 2024-11-21T21:24:32,375 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:32,375 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224332382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:32,418 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121269e37c3c7624dde87b5d5b165a64317_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121269e37c3c7624dde87b5d5b165a64317_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:32,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/9d3569fe2f8e48df92d6eb65b636c7c6, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:32,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/9d3569fe2f8e48df92d6eb65b636c7c6 is 175, key is test_row_0/A:col10/1732224270851/Put/seqid=0 2024-11-21T21:24:32,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741984_1160 (size=30955) 2024-11-21T21:24:32,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224332660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224332660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224332669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224332689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:24:32,844 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/9d3569fe2f8e48df92d6eb65b636c7c6 2024-11-21T21:24:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/cb47067ce7294763a9ede23c2dbdc2e7 is 50, key is test_row_0/B:col10/1732224270851/Put/seqid=0 2024-11-21T21:24:32,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741985_1161 (size=12001) 2024-11-21T21:24:32,905 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/cb47067ce7294763a9ede23c2dbdc2e7 2024-11-21T21:24:32,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/8db0fbca48b3462c910b48852097721f is 50, key is test_row_0/C:col10/1732224270851/Put/seqid=0 2024-11-21T21:24:32,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741986_1162 (size=12001) 2024-11-21T21:24:32,961 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/8db0fbca48b3462c910b48852097721f 2024-11-21T21:24:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/9d3569fe2f8e48df92d6eb65b636c7c6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/9d3569fe2f8e48df92d6eb65b636c7c6 2024-11-21T21:24:32,977 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/9d3569fe2f8e48df92d6eb65b636c7c6, entries=150, sequenceid=128, filesize=30.2 K 2024-11-21T21:24:32,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/cb47067ce7294763a9ede23c2dbdc2e7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb47067ce7294763a9ede23c2dbdc2e7 2024-11-21T21:24:32,992 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb47067ce7294763a9ede23c2dbdc2e7, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T21:24:32,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/8db0fbca48b3462c910b48852097721f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8db0fbca48b3462c910b48852097721f 2024-11-21T21:24:32,999 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8db0fbca48b3462c910b48852097721f, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T21:24:33,001 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for cd5da8e0c287760dca4b3d875dd531a4 in 1077ms, sequenceid=128, compaction requested=false 2024-11-21T21:24:33,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:33,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:33,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T21:24:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T21:24:33,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T21:24:33,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3130 sec 2024-11-21T21:24:33,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3240 sec 2024-11-21T21:24:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:33,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:24:33,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112159110da5491740bf869002df365f9ea7_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224273050/Put/seqid=0 2024-11-21T21:24:33,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741987_1163 (size=14794) 2024-11-21T21:24:33,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224333146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224333170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224333180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224333180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224333199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224333250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224333459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:33,534 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112159110da5491740bf869002df365f9ea7_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112159110da5491740bf869002df365f9ea7_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:33,535 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/0587a2ae33424d67a2a0a0c2d2555c10, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:33,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/0587a2ae33424d67a2a0a0c2d2555c10 is 175, key is test_row_0/A:col10/1732224273050/Put/seqid=0 2024-11-21T21:24:33,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741988_1164 (size=39749) 2024-11-21T21:24:33,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/0587a2ae33424d67a2a0a0c2d2555c10 2024-11-21T21:24:33,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/88aeab420fd9415bb05a238e07c46bfd is 50, key is test_row_0/B:col10/1732224273050/Put/seqid=0 2024-11-21T21:24:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741989_1165 (size=12151) 2024-11-21T21:24:33,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/88aeab420fd9415bb05a238e07c46bfd 2024-11-21T21:24:33,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/87b035e93d554e62a5dc5a37b99c4494 is 50, key is test_row_0/C:col10/1732224273050/Put/seqid=0 2024-11-21T21:24:33,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741990_1166 (size=12151) 2024-11-21T21:24:33,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/87b035e93d554e62a5dc5a37b99c4494 2024-11-21T21:24:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/0587a2ae33424d67a2a0a0c2d2555c10 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0587a2ae33424d67a2a0a0c2d2555c10 2024-11-21T21:24:33,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0587a2ae33424d67a2a0a0c2d2555c10, entries=200, sequenceid=143, filesize=38.8 K 2024-11-21T21:24:33,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/88aeab420fd9415bb05a238e07c46bfd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/88aeab420fd9415bb05a238e07c46bfd 2024-11-21T21:24:33,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/88aeab420fd9415bb05a238e07c46bfd, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T21:24:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/87b035e93d554e62a5dc5a37b99c4494 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/87b035e93d554e62a5dc5a37b99c4494 2024-11-21T21:24:33,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/87b035e93d554e62a5dc5a37b99c4494, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T21:24:33,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cd5da8e0c287760dca4b3d875dd531a4 in 695ms, sequenceid=143, compaction requested=true 2024-11-21T21:24:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:33,746 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:33,747 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:33,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:33,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:33,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:33,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:33,751 INFO [master/849dee24a242:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T21:24:33,751 INFO [master/849dee24a242:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T21:24:33,752 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:33,753 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:33,753 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:33,753 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0d9095df52d74a0ab55429ed167b7c14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/9d3569fe2f8e48df92d6eb65b636c7c6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0587a2ae33424d67a2a0a0c2d2555c10] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=99.5 K 2024-11-21T21:24:33,753 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:33,753 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0d9095df52d74a0ab55429ed167b7c14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/9d3569fe2f8e48df92d6eb65b636c7c6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0587a2ae33424d67a2a0a0c2d2555c10] 2024-11-21T21:24:33,753 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:33,753 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:33,754 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:33,754 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f8c96d84f24f4d54941ecdbcde912cea, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb47067ce7294763a9ede23c2dbdc2e7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/88aeab420fd9415bb05a238e07c46bfd] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=35.5 K 2024-11-21T21:24:33,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d9095df52d74a0ab55429ed167b7c14, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732224270784 2024-11-21T21:24:33,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c96d84f24f4d54941ecdbcde912cea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732224270784 2024-11-21T21:24:33,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d3569fe2f8e48df92d6eb65b636c7c6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732224270851 2024-11-21T21:24:33,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting cb47067ce7294763a9ede23c2dbdc2e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732224270851 2024-11-21T21:24:33,755 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0587a2ae33424d67a2a0a0c2d2555c10, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732224272027 2024-11-21T21:24:33,755 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 88aeab420fd9415bb05a238e07c46bfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732224272027 2024-11-21T21:24:33,764 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:33,770 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:33,770 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/090c6187b42f48ea943802eb729daef7 is 50, key is test_row_0/B:col10/1732224273050/Put/seqid=0 2024-11-21T21:24:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:33,782 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211f5a968c44cc42b597d3b75f8d75d03a_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:33,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:24:33,784 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211f5a968c44cc42b597d3b75f8d75d03a_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:33,785 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211f5a968c44cc42b597d3b75f8d75d03a_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:33,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112163da21cd3ffd439d8f9653a67da32d34_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224273780/Put/seqid=0 2024-11-21T21:24:33,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741992_1168 (size=4469) 2024-11-21T21:24:33,826 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#141 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:33,827 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/333f67fc99314262af9601fa73b2fbdf is 175, key is test_row_0/A:col10/1732224273050/Put/seqid=0 2024-11-21T21:24:33,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741991_1167 (size=12493) 2024-11-21T21:24:33,842 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/090c6187b42f48ea943802eb729daef7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/090c6187b42f48ea943802eb729daef7 2024-11-21T21:24:33,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224333843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:33,850 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into 090c6187b42f48ea943802eb729daef7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:33,850 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:33,850 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=13, startTime=1732224273746; duration=0sec 2024-11-21T21:24:33,850 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:33,850 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:33,850 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:33,851 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:33,851 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:33,851 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:33,852 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/e96ca9bd774f4817967e86796b99388f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8db0fbca48b3462c910b48852097721f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/87b035e93d554e62a5dc5a37b99c4494] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=35.5 K 2024-11-21T21:24:33,852 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e96ca9bd774f4817967e86796b99388f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732224270784 2024-11-21T21:24:33,853 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db0fbca48b3462c910b48852097721f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732224270851 2024-11-21T21:24:33,853 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b035e93d554e62a5dc5a37b99c4494, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732224272027 2024-11-21T21:24:33,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741993_1169 (size=14794) 2024-11-21T21:24:33,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:33,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112163da21cd3ffd439d8f9653a67da32d34_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112163da21cd3ffd439d8f9653a67da32d34_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:33,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/05ecce48816f46c49bcd295d868fede1, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:33,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/05ecce48816f46c49bcd295d868fede1 is 175, key is test_row_0/A:col10/1732224273780/Put/seqid=0 2024-11-21T21:24:33,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741994_1170 (size=31447) 2024-11-21T21:24:33,888 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:33,889 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/333f67fc99314262af9601fa73b2fbdf as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/333f67fc99314262af9601fa73b2fbdf 2024-11-21T21:24:33,889 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/f2e6f498f2824b0c890aa1fffad0dd44 is 50, key is test_row_0/C:col10/1732224273050/Put/seqid=0 2024-11-21T21:24:33,897 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into 333f67fc99314262af9601fa73b2fbdf(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:33,897 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:33,897 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=13, startTime=1732224273746; duration=0sec 2024-11-21T21:24:33,898 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:33,898 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:33,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741995_1171 (size=39749) 2024-11-21T21:24:33,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/05ecce48816f46c49bcd295d868fede1 2024-11-21T21:24:33,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741996_1172 (size=12493) 2024-11-21T21:24:33,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/578e748ca8b94701b85b2a0b32f19de0 is 50, key is test_row_0/B:col10/1732224273780/Put/seqid=0 2024-11-21T21:24:33,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741997_1173 (size=12151) 2024-11-21T21:24:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224333960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:34,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224334180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:34,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224334189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:34,196 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:34,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224334199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:34,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224334200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:34,201 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:34,202 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:34,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224334210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:34,217 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:34,337 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/f2e6f498f2824b0c890aa1fffad0dd44 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f2e6f498f2824b0c890aa1fffad0dd44 2024-11-21T21:24:34,348 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into f2e6f498f2824b0c890aa1fffad0dd44(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:34,348 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:34,348 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=13, startTime=1732224273747; duration=0sec 2024-11-21T21:24:34,352 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:34,352 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:34,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/578e748ca8b94701b85b2a0b32f19de0 2024-11-21T21:24:34,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/d3866d3d0ac44ea788247a98ddf8abcf is 50, key is test_row_0/C:col10/1732224273780/Put/seqid=0 2024-11-21T21:24:34,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741998_1174 (size=12151) 2024-11-21T21:24:34,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224334490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:34,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/d3866d3d0ac44ea788247a98ddf8abcf 2024-11-21T21:24:34,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/05ecce48816f46c49bcd295d868fede1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/05ecce48816f46c49bcd295d868fede1 2024-11-21T21:24:34,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/05ecce48816f46c49bcd295d868fede1, entries=200, sequenceid=167, filesize=38.8 K 2024-11-21T21:24:34,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/578e748ca8b94701b85b2a0b32f19de0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/578e748ca8b94701b85b2a0b32f19de0 2024-11-21T21:24:34,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/578e748ca8b94701b85b2a0b32f19de0, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T21:24:34,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/d3866d3d0ac44ea788247a98ddf8abcf as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/d3866d3d0ac44ea788247a98ddf8abcf 2024-11-21T21:24:34,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/d3866d3d0ac44ea788247a98ddf8abcf, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T21:24:34,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for cd5da8e0c287760dca4b3d875dd531a4 in 1040ms, sequenceid=167, compaction requested=false 2024-11-21T21:24:34,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:24:34,850 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:34,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T21:24:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:24:34,852 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:34,853 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:34,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:24:35,004 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:35,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:24:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:35,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:35,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:35,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218c7abceb4a93472b943a0628a7ea415a_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224273820/Put/seqid=0 2024-11-21T21:24:35,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741999_1175 (size=12304) 2024-11-21T21:24:35,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:35,083 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218c7abceb4a93472b943a0628a7ea415a_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218c7abceb4a93472b943a0628a7ea415a_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:35,084 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7edfd81747c743e994de4c1c648926b2, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:35,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7edfd81747c743e994de4c1c648926b2 is 175, key is test_row_0/A:col10/1732224273820/Put/seqid=0 2024-11-21T21:24:35,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742000_1176 (size=31105) 2024-11-21T21:24:35,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7edfd81747c743e994de4c1c648926b2 2024-11-21T21:24:35,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224335123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:35,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/06ca25c7e4644bcf91c7757cd8e7235b is 50, key is test_row_0/B:col10/1732224273820/Put/seqid=0 2024-11-21T21:24:35,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:24:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742001_1177 (size=12151) 2024-11-21T21:24:35,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/06ca25c7e4644bcf91c7757cd8e7235b 2024-11-21T21:24:35,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/06e7f3cb4e954a6baa6f067e9f23256f is 50, key is test_row_0/C:col10/1732224273820/Put/seqid=0 2024-11-21T21:24:35,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742002_1178 (size=12151) 2024-11-21T21:24:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224335240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:35,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:35,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:35,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224335449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:35,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:35,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:35,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:24:35,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/06e7f3cb4e954a6baa6f067e9f23256f 2024-11-21T21:24:35,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7edfd81747c743e994de4c1c648926b2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7edfd81747c743e994de4c1c648926b2 2024-11-21T21:24:35,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7edfd81747c743e994de4c1c648926b2, entries=150, sequenceid=183, filesize=30.4 K 2024-11-21T21:24:35,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/06ca25c7e4644bcf91c7757cd8e7235b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/06ca25c7e4644bcf91c7757cd8e7235b 2024-11-21T21:24:35,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/06ca25c7e4644bcf91c7757cd8e7235b, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T21:24:35,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/06e7f3cb4e954a6baa6f067e9f23256f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/06e7f3cb4e954a6baa6f067e9f23256f 2024-11-21T21:24:35,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/06e7f3cb4e954a6baa6f067e9f23256f, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T21:24:35,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cd5da8e0c287760dca4b3d875dd531a4 in 638ms, sequenceid=183, compaction requested=true 2024-11-21T21:24:35,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:35,642 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:35,643 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:35,643 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:35,643 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:35,644 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,644 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/333f67fc99314262af9601fa73b2fbdf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/05ecce48816f46c49bcd295d868fede1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7edfd81747c743e994de4c1c648926b2] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=99.9 K 2024-11-21T21:24:35,644 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:35,644 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/333f67fc99314262af9601fa73b2fbdf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/05ecce48816f46c49bcd295d868fede1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7edfd81747c743e994de4c1c648926b2] 2024-11-21T21:24:35,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:35,644 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 333f67fc99314262af9601fa73b2fbdf, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732224272027 2024-11-21T21:24:35,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:35,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:35,645 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:35,645 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ecce48816f46c49bcd295d868fede1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732224273117 2024-11-21T21:24:35,645 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:35,645 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,645 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/090c6187b42f48ea943802eb729daef7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/578e748ca8b94701b85b2a0b32f19de0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/06ca25c7e4644bcf91c7757cd8e7235b] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=35.9 K 2024-11-21T21:24:35,645 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7edfd81747c743e994de4c1c648926b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732224273820 2024-11-21T21:24:35,645 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 090c6187b42f48ea943802eb729daef7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732224272027 2024-11-21T21:24:35,646 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 578e748ca8b94701b85b2a0b32f19de0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732224273117 2024-11-21T21:24:35,647 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ca25c7e4644bcf91c7757cd8e7235b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732224273820 2024-11-21T21:24:35,654 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:35,669 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217f4974a686cb45e7aa6f37e7edf6feb9_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:35,671 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217f4974a686cb45e7aa6f37e7edf6feb9_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:35,671 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217f4974a686cb45e7aa6f37e7edf6feb9_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:35,678 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:35,678 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/d6db316d639e4a5e8451a8ff04b89616 is 50, key is test_row_0/B:col10/1732224273820/Put/seqid=0 2024-11-21T21:24:35,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742003_1179 (size=4469) 2024-11-21T21:24:35,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742004_1180 (size=12595) 2024-11-21T21:24:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:35,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:35,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:24:35,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121319ddf843aa64652a075f0afcdf05f3b_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224275097/Put/seqid=0 2024-11-21T21:24:35,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742005_1181 (size=14794) 2024-11-21T21:24:35,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:35,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,786 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:35,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121319ddf843aa64652a075f0afcdf05f3b_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121319ddf843aa64652a075f0afcdf05f3b_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:35,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224335797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:35,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/02389cf6249248f6a9ea17b5738dbd59, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:35,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/02389cf6249248f6a9ea17b5738dbd59 is 175, key is test_row_0/A:col10/1732224275097/Put/seqid=0 2024-11-21T21:24:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742006_1182 (size=39749) 2024-11-21T21:24:35,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/02389cf6249248f6a9ea17b5738dbd59 2024-11-21T21:24:35,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2356493c88b24db999ebd8924a4bbbaf is 50, key is test_row_0/B:col10/1732224275097/Put/seqid=0 2024-11-21T21:24:35,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742007_1183 (size=12151) 2024-11-21T21:24:35,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224335909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:35,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:35,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:24:36,084 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:36,085 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/468b757ee4e242c4b0dbfd4e1a66b265 is 175, key is test_row_0/A:col10/1732224273820/Put/seqid=0 2024-11-21T21:24:36,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:36,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:36,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,110 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/d6db316d639e4a5e8451a8ff04b89616 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/d6db316d639e4a5e8451a8ff04b89616 2024-11-21T21:24:36,117 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into d6db316d639e4a5e8451a8ff04b89616(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:36,117 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:36,117 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=13, startTime=1732224275643; duration=0sec 2024-11-21T21:24:36,117 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:36,118 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:36,118 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:36,120 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:36,120 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:36,120 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,120 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f2e6f498f2824b0c890aa1fffad0dd44, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/d3866d3d0ac44ea788247a98ddf8abcf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/06e7f3cb4e954a6baa6f067e9f23256f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=35.9 K 2024-11-21T21:24:36,121 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e6f498f2824b0c890aa1fffad0dd44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732224272027 2024-11-21T21:24:36,121 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting d3866d3d0ac44ea788247a98ddf8abcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732224273117 2024-11-21T21:24:36,122 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e7f3cb4e954a6baa6f067e9f23256f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732224273820 2024-11-21T21:24:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224336119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:36,131 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#154 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:36,131 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/ab1af2ef2d5343faa4a662aff1fef6bc is 50, key is test_row_0/C:col10/1732224273820/Put/seqid=0 2024-11-21T21:24:36,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742008_1184 (size=31549) 2024-11-21T21:24:36,151 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/468b757ee4e242c4b0dbfd4e1a66b265 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/468b757ee4e242c4b0dbfd4e1a66b265 2024-11-21T21:24:36,164 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into 468b757ee4e242c4b0dbfd4e1a66b265(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:36,164 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:36,164 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=13, startTime=1732224275642; duration=0sec 2024-11-21T21:24:36,164 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:36,164 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:36,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742009_1185 (size=12595) 2024-11-21T21:24:36,189 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/ab1af2ef2d5343faa4a662aff1fef6bc as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/ab1af2ef2d5343faa4a662aff1fef6bc 2024-11-21T21:24:36,195 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into ab1af2ef2d5343faa4a662aff1fef6bc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:36,195 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:36,195 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=13, startTime=1732224275644; duration=0sec 2024-11-21T21:24:36,196 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:36,196 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224336210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:36,210 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:36,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:36,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224336220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:36,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224336221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:36,222 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:36,223 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224336240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:36,240 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:36,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2356493c88b24db999ebd8924a4bbbaf 2024-11-21T21:24:36,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/558bc66a39144d13a8948307f21e1c17 is 50, key is test_row_0/C:col10/1732224275097/Put/seqid=0 2024-11-21T21:24:36,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742010_1186 (size=12151) 2024-11-21T21:24:36,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224336429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:36,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:36,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/558bc66a39144d13a8948307f21e1c17 2024-11-21T21:24:36,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/02389cf6249248f6a9ea17b5738dbd59 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/02389cf6249248f6a9ea17b5738dbd59 2024-11-21T21:24:36,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/02389cf6249248f6a9ea17b5738dbd59, entries=200, sequenceid=207, filesize=38.8 K 2024-11-21T21:24:36,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2356493c88b24db999ebd8924a4bbbaf as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2356493c88b24db999ebd8924a4bbbaf 2024-11-21T21:24:36,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2356493c88b24db999ebd8924a4bbbaf, entries=150, sequenceid=207, filesize=11.9 K 2024-11-21T21:24:36,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/558bc66a39144d13a8948307f21e1c17 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/558bc66a39144d13a8948307f21e1c17 2024-11-21T21:24:36,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/558bc66a39144d13a8948307f21e1c17, entries=150, sequenceid=207, filesize=11.9 K 2024-11-21T21:24:36,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for cd5da8e0c287760dca4b3d875dd531a4 in 934ms, sequenceid=207, compaction requested=false 2024-11-21T21:24:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:36,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:24:36,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:36,703 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:24:36,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a93923685fa24f90913287fe99a61d37_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224275795/Put/seqid=0 2024-11-21T21:24:36,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742011_1187 (size=12304) 2024-11-21T21:24:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:36,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:24:37,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224337067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:37,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:37,132 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a93923685fa24f90913287fe99a61d37_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a93923685fa24f90913287fe99a61d37_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7193820220de4f098c575f9e0fa3fd76, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7193820220de4f098c575f9e0fa3fd76 is 175, key is test_row_0/A:col10/1732224275795/Put/seqid=0 2024-11-21T21:24:37,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742012_1188 (size=31105) 2024-11-21T21:24:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224337180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:37,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224337390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:37,559 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7193820220de4f098c575f9e0fa3fd76 2024-11-21T21:24:37,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2f2a6c5e670e46ccb98ad3c9e934e753 is 50, key is test_row_0/B:col10/1732224275795/Put/seqid=0 2024-11-21T21:24:37,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742013_1189 (size=12151) 2024-11-21T21:24:37,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224337699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:37,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:37,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:37,993 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2f2a6c5e670e46ccb98ad3c9e934e753 2024-11-21T21:24:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/97a159f8498149f5a6c9d21a8d93ab1c is 50, key is test_row_0/C:col10/1732224275795/Put/seqid=0 2024-11-21T21:24:38,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742014_1190 (size=12151) 2024-11-21T21:24:38,054 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/97a159f8498149f5a6c9d21a8d93ab1c 2024-11-21T21:24:38,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7193820220de4f098c575f9e0fa3fd76 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7193820220de4f098c575f9e0fa3fd76 2024-11-21T21:24:38,069 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7193820220de4f098c575f9e0fa3fd76, entries=150, sequenceid=222, filesize=30.4 K 2024-11-21T21:24:38,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T21:24:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2f2a6c5e670e46ccb98ad3c9e934e753 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2f2a6c5e670e46ccb98ad3c9e934e753 2024-11-21T21:24:38,077 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2f2a6c5e670e46ccb98ad3c9e934e753, entries=150, sequenceid=222, filesize=11.9 K 2024-11-21T21:24:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/97a159f8498149f5a6c9d21a8d93ab1c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/97a159f8498149f5a6c9d21a8d93ab1c 2024-11-21T21:24:38,083 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/97a159f8498149f5a6c9d21a8d93ab1c, entries=150, sequenceid=222, filesize=11.9 K 2024-11-21T21:24:38,085 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cd5da8e0c287760dca4b3d875dd531a4 in 1381ms, sequenceid=222, compaction requested=true 2024-11-21T21:24:38,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:38,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:38,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T21:24:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T21:24:38,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T21:24:38,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2340 sec 2024-11-21T21:24:38,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.2400 sec 2024-11-21T21:24:38,226 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:38,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:24:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:38,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215f171481c4614367980c96a7f20476c4_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224278222/Put/seqid=0 2024-11-21T21:24:38,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224338268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:38,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742015_1191 (size=14794) 2024-11-21T21:24:38,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:38,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215f171481c4614367980c96a7f20476c4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215f171481c4614367980c96a7f20476c4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:38,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/4b23457c6a084d349b298272cc5b5125, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:38,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/4b23457c6a084d349b298272cc5b5125 is 175, key is test_row_0/A:col10/1732224278222/Put/seqid=0 2024-11-21T21:24:38,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742016_1192 (size=39749) 2024-11-21T21:24:38,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224338379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224338589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:38,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/4b23457c6a084d349b298272cc5b5125 2024-11-21T21:24:38,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/1e83c0c0a0fb4c6fb2efe9780a518e8c is 50, key is test_row_0/B:col10/1732224278222/Put/seqid=0 2024-11-21T21:24:38,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742017_1193 (size=12151) 2024-11-21T21:24:38,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224338899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:24:39,020 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:39,020 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T21:24:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:24:39,024 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:39,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:39,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:24:39,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/1e83c0c0a0fb4c6fb2efe9780a518e8c 2024-11-21T21:24:39,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/9f94f9a21cf8402790226892d85558a1 is 50, key is test_row_0/C:col10/1732224278222/Put/seqid=0 2024-11-21T21:24:39,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742018_1194 (size=12151) 2024-11-21T21:24:39,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:24:39,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:39,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:24:39,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:39,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:24:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224339409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:39,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:24:39,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:39,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:39,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/9f94f9a21cf8402790226892d85558a1 2024-11-21T21:24:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/4b23457c6a084d349b298272cc5b5125 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/4b23457c6a084d349b298272cc5b5125 2024-11-21T21:24:39,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/4b23457c6a084d349b298272cc5b5125, entries=200, sequenceid=247, filesize=38.8 K 2024-11-21T21:24:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/1e83c0c0a0fb4c6fb2efe9780a518e8c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/1e83c0c0a0fb4c6fb2efe9780a518e8c 2024-11-21T21:24:39,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/1e83c0c0a0fb4c6fb2efe9780a518e8c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T21:24:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/9f94f9a21cf8402790226892d85558a1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9f94f9a21cf8402790226892d85558a1 2024-11-21T21:24:39,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9f94f9a21cf8402790226892d85558a1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T21:24:39,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for cd5da8e0c287760dca4b3d875dd531a4 in 1353ms, sequenceid=247, compaction requested=true 2024-11-21T21:24:39,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:39,580 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:39,580 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:39,581 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:39,581 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:39,581 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,582 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/468b757ee4e242c4b0dbfd4e1a66b265, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/02389cf6249248f6a9ea17b5738dbd59, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7193820220de4f098c575f9e0fa3fd76, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/4b23457c6a084d349b298272cc5b5125] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=138.8 K 2024-11-21T21:24:39,582 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,582 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/468b757ee4e242c4b0dbfd4e1a66b265, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/02389cf6249248f6a9ea17b5738dbd59, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7193820220de4f098c575f9e0fa3fd76, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/4b23457c6a084d349b298272cc5b5125] 2024-11-21T21:24:39,582 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 468b757ee4e242c4b0dbfd4e1a66b265, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732224273820 2024-11-21T21:24:39,582 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:39,582 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:39,582 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,582 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/d6db316d639e4a5e8451a8ff04b89616, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2356493c88b24db999ebd8924a4bbbaf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2f2a6c5e670e46ccb98ad3c9e934e753, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/1e83c0c0a0fb4c6fb2efe9780a518e8c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=47.9 K 2024-11-21T21:24:39,582 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02389cf6249248f6a9ea17b5738dbd59, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732224275097 2024-11-21T21:24:39,583 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting d6db316d639e4a5e8451a8ff04b89616, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732224273820 2024-11-21T21:24:39,583 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7193820220de4f098c575f9e0fa3fd76, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732224275784 2024-11-21T21:24:39,583 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b23457c6a084d349b298272cc5b5125, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732224277017 2024-11-21T21:24:39,583 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2356493c88b24db999ebd8924a4bbbaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732224275097 2024-11-21T21:24:39,583 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2a6c5e670e46ccb98ad3c9e934e753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732224275784 2024-11-21T21:24:39,584 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e83c0c0a0fb4c6fb2efe9780a518e8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732224277017 2024-11-21T21:24:39,590 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:39,595 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:39,596 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/8434250f9d0f4735bdc5206eb6a315d8 is 50, key is test_row_0/B:col10/1732224278222/Put/seqid=0 2024-11-21T21:24:39,597 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411218897aadc809142baaa24fafa35bf21c9_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:39,599 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411218897aadc809142baaa24fafa35bf21c9_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:39,600 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218897aadc809142baaa24fafa35bf21c9_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:39,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742019_1195 (size=12731) 2024-11-21T21:24:39,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742020_1196 (size=4469) 2024-11-21T21:24:39,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:24:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:39,637 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:24:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bdfe61723c2246fd8e9517f2f4ca4023_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224278231/Put/seqid=0 2024-11-21T21:24:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:24:39,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742021_1197 (size=12354) 2024-11-21T21:24:40,009 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/8434250f9d0f4735bdc5206eb6a315d8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/8434250f9d0f4735bdc5206eb6a315d8 2024-11-21T21:24:40,013 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:40,013 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/48a317a02d204d19977f80c5214562a4 is 175, key is test_row_0/A:col10/1732224278222/Put/seqid=0 2024-11-21T21:24:40,015 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into 8434250f9d0f4735bdc5206eb6a315d8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:40,015 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:40,015 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=12, startTime=1732224279580; duration=0sec 2024-11-21T21:24:40,015 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:40,015 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:40,015 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:40,017 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:40,017 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:40,017 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742022_1198 (size=31685) 2024-11-21T21:24:40,017 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/ab1af2ef2d5343faa4a662aff1fef6bc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/558bc66a39144d13a8948307f21e1c17, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/97a159f8498149f5a6c9d21a8d93ab1c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9f94f9a21cf8402790226892d85558a1] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=47.9 K 2024-11-21T21:24:40,018 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ab1af2ef2d5343faa4a662aff1fef6bc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732224273820 2024-11-21T21:24:40,018 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 558bc66a39144d13a8948307f21e1c17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732224275097 2024-11-21T21:24:40,018 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 97a159f8498149f5a6c9d21a8d93ab1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732224275784 2024-11-21T21:24:40,019 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f94f9a21cf8402790226892d85558a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732224277017 2024-11-21T21:24:40,023 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/48a317a02d204d19977f80c5214562a4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/48a317a02d204d19977f80c5214562a4 2024-11-21T21:24:40,029 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into 48a317a02d204d19977f80c5214562a4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:40,029 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:40,029 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=12, startTime=1732224279579; duration=0sec 2024-11-21T21:24:40,029 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:40,029 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:40,029 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:40,030 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/35dca5d5d1954c9b9fd2c76ce125a02c is 50, key is test_row_0/C:col10/1732224278222/Put/seqid=0 2024-11-21T21:24:40,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742023_1199 (size=12731) 2024-11-21T21:24:40,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:40,057 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bdfe61723c2246fd8e9517f2f4ca4023_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bdfe61723c2246fd8e9517f2f4ca4023_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:40,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/f71d151386164e3f95bf922074843587, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/f71d151386164e3f95bf922074843587 is 175, key is test_row_0/A:col10/1732224278231/Put/seqid=0 2024-11-21T21:24:40,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742024_1200 (size=31155) 2024-11-21T21:24:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:24:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:40,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:40,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224340292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224340327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224340327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224340328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224340432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224340432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,437 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224340440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224340440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224340440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,446 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/35dca5d5d1954c9b9fd2c76ce125a02c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/35dca5d5d1954c9b9fd2c76ce125a02c 2024-11-21T21:24:40,451 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into 35dca5d5d1954c9b9fd2c76ce125a02c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:40,451 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:40,451 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=12, startTime=1732224279580; duration=0sec 2024-11-21T21:24:40,452 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:40,452 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:40,466 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/f71d151386164e3f95bf922074843587 2024-11-21T21:24:40,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/f2a18c1801054380aa68e59b60a45193 is 50, key is test_row_0/B:col10/1732224278231/Put/seqid=0 2024-11-21T21:24:40,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742025_1201 (size=12201) 2024-11-21T21:24:40,506 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/f2a18c1801054380aa68e59b60a45193 2024-11-21T21:24:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/758adc1a08b84ca0bcb4f51435f4b6b2 is 50, key is test_row_0/C:col10/1732224278231/Put/seqid=0 2024-11-21T21:24:40,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742026_1202 (size=12201) 2024-11-21T21:24:40,533 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/758adc1a08b84ca0bcb4f51435f4b6b2 2024-11-21T21:24:40,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/f71d151386164e3f95bf922074843587 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/f71d151386164e3f95bf922074843587 2024-11-21T21:24:40,545 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/f71d151386164e3f95bf922074843587, entries=150, sequenceid=258, filesize=30.4 K 2024-11-21T21:24:40,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/f2a18c1801054380aa68e59b60a45193 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f2a18c1801054380aa68e59b60a45193 2024-11-21T21:24:40,553 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f2a18c1801054380aa68e59b60a45193, entries=150, sequenceid=258, filesize=11.9 K 2024-11-21T21:24:40,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/758adc1a08b84ca0bcb4f51435f4b6b2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/758adc1a08b84ca0bcb4f51435f4b6b2 2024-11-21T21:24:40,564 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/758adc1a08b84ca0bcb4f51435f4b6b2, entries=150, sequenceid=258, filesize=11.9 K 2024-11-21T21:24:40,566 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cd5da8e0c287760dca4b3d875dd531a4 in 930ms, sequenceid=258, compaction requested=false 2024-11-21T21:24:40,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:40,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:40,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T21:24:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T21:24:40,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T21:24:40,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5430 sec 2024-11-21T21:24:40,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5510 sec 2024-11-21T21:24:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:40,645 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:40,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T21:24:40,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224340658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224340658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224340660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224340663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121179ea192120f477982eab4a0b6b4722a_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224280291/Put/seqid=0 2024-11-21T21:24:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742027_1203 (size=12454) 2024-11-21T21:24:40,679 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:40,685 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121179ea192120f477982eab4a0b6b4722a_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121179ea192120f477982eab4a0b6b4722a_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:40,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b836f49f6e5f42a1afaa9d3a2a1f4d66, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:40,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b836f49f6e5f42a1afaa9d3a2a1f4d66 is 175, key is test_row_0/A:col10/1732224280291/Put/seqid=0 2024-11-21T21:24:40,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742028_1204 (size=31255) 2024-11-21T21:24:40,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b836f49f6e5f42a1afaa9d3a2a1f4d66 2024-11-21T21:24:40,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/cb75cacdf04e4af9b58b3880df513c39 is 50, key is test_row_0/B:col10/1732224280291/Put/seqid=0 2024-11-21T21:24:40,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742029_1205 (size=12301) 2024-11-21T21:24:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224340769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224340769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224340769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224340780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224340979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224340979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224340979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:40,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224340990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/cb75cacdf04e4af9b58b3880df513c39 2024-11-21T21:24:41,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/41196276a28a4145b8667912613be00c is 50, key is test_row_0/C:col10/1732224280291/Put/seqid=0 2024-11-21T21:24:41,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742030_1206 (size=12301) 2024-11-21T21:24:41,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/41196276a28a4145b8667912613be00c 2024-11-21T21:24:41,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b836f49f6e5f42a1afaa9d3a2a1f4d66 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b836f49f6e5f42a1afaa9d3a2a1f4d66 2024-11-21T21:24:41,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b836f49f6e5f42a1afaa9d3a2a1f4d66, entries=150, sequenceid=287, filesize=30.5 K 2024-11-21T21:24:41,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/cb75cacdf04e4af9b58b3880df513c39 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb75cacdf04e4af9b58b3880df513c39 2024-11-21T21:24:41,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb75cacdf04e4af9b58b3880df513c39, entries=150, sequenceid=287, filesize=12.0 K 2024-11-21T21:24:41,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/41196276a28a4145b8667912613be00c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/41196276a28a4145b8667912613be00c 2024-11-21T21:24:41,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/41196276a28a4145b8667912613be00c, entries=150, sequenceid=287, filesize=12.0 K 2024-11-21T21:24:41,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for cd5da8e0c287760dca4b3d875dd531a4 in 518ms, sequenceid=287, compaction requested=true 2024-11-21T21:24:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:41,164 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:41,164 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:41,165 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:41,165 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:41,165 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:41,165 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:41,165 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,165 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,165 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/48a317a02d204d19977f80c5214562a4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/f71d151386164e3f95bf922074843587, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b836f49f6e5f42a1afaa9d3a2a1f4d66] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=91.9 K 2024-11-21T21:24:41,165 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/8434250f9d0f4735bdc5206eb6a315d8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f2a18c1801054380aa68e59b60a45193, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb75cacdf04e4af9b58b3880df513c39] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=36.4 K 2024-11-21T21:24:41,165 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,165 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/48a317a02d204d19977f80c5214562a4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/f71d151386164e3f95bf922074843587, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b836f49f6e5f42a1afaa9d3a2a1f4d66] 2024-11-21T21:24:41,165 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a317a02d204d19977f80c5214562a4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732224277017 2024-11-21T21:24:41,165 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8434250f9d0f4735bdc5206eb6a315d8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732224277017 2024-11-21T21:24:41,166 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f71d151386164e3f95bf922074843587, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732224278231 2024-11-21T21:24:41,166 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a18c1801054380aa68e59b60a45193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732224278231 2024-11-21T21:24:41,166 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b836f49f6e5f42a1afaa9d3a2a1f4d66, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732224280291 2024-11-21T21:24:41,166 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting cb75cacdf04e4af9b58b3880df513c39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732224280291 2024-11-21T21:24:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:24:41,169 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:41,170 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T21:24:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:24:41,172 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:41,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:41,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:41,177 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:41,179 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210c923b71eaa34b13a21efd3ba12604fe_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:41,180 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:41,180 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/83bf59be1bc14a37bad6f4d30ed1942b is 50, key is test_row_0/B:col10/1732224280291/Put/seqid=0 2024-11-21T21:24:41,181 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210c923b71eaa34b13a21efd3ba12604fe_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:41,181 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210c923b71eaa34b13a21efd3ba12604fe_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:41,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742031_1207 (size=12983) 2024-11-21T21:24:41,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742032_1208 (size=4469) 2024-11-21T21:24:41,218 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#171 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:41,219 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/93d5a2ab663f4fa9bff17d8cc7c41dd0 is 175, key is test_row_0/A:col10/1732224280291/Put/seqid=0 2024-11-21T21:24:41,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742033_1209 (size=31937) 2024-11-21T21:24:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:24:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:41,291 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:41,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:24:41,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c3630a260dc34d2b97e6b9b125971710_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224281290/Put/seqid=0 2024-11-21T21:24:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742034_1210 (size=17534) 2024-11-21T21:24:41,318 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:41,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T21:24:41,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c3630a260dc34d2b97e6b9b125971710_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c3630a260dc34d2b97e6b9b125971710_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:41,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:41,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,328 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/bed91d42e8e440f7ae92259356f10e85, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:41,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/bed91d42e8e440f7ae92259356f10e85 is 175, key is test_row_0/A:col10/1732224281290/Put/seqid=0 2024-11-21T21:24:41,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742035_1211 (size=48639) 2024-11-21T21:24:41,335 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/bed91d42e8e440f7ae92259356f10e85 2024-11-21T21:24:41,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224341333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224341335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/3db9359dff1c48ddac37f2172b01de54 is 50, key is test_row_0/B:col10/1732224281290/Put/seqid=0 2024-11-21T21:24:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224341343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224341343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742036_1212 (size=12301) 2024-11-21T21:24:41,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/3db9359dff1c48ddac37f2172b01de54 2024-11-21T21:24:41,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/0534f5307967415ab09ea44133b8626a is 50, key is test_row_0/C:col10/1732224281290/Put/seqid=0 2024-11-21T21:24:41,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742037_1213 (size=12301) 2024-11-21T21:24:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224341450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224341450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224341459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224341459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T21:24:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:24:41,599 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/83bf59be1bc14a37bad6f4d30ed1942b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/83bf59be1bc14a37bad6f4d30ed1942b 2024-11-21T21:24:41,604 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into 83bf59be1bc14a37bad6f4d30ed1942b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:41,604 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:41,604 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=13, startTime=1732224281164; duration=0sec 2024-11-21T21:24:41,604 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:41,604 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:41,604 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:41,605 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:41,605 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:41,605 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,605 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/35dca5d5d1954c9b9fd2c76ce125a02c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/758adc1a08b84ca0bcb4f51435f4b6b2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/41196276a28a4145b8667912613be00c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=36.4 K 2024-11-21T21:24:41,606 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 35dca5d5d1954c9b9fd2c76ce125a02c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732224277017 2024-11-21T21:24:41,606 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 758adc1a08b84ca0bcb4f51435f4b6b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732224278231 2024-11-21T21:24:41,606 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 41196276a28a4145b8667912613be00c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732224280291 2024-11-21T21:24:41,612 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:41,612 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/92ceb6089d0f45d19ebdb65ab3fb3c5c is 50, key is test_row_0/C:col10/1732224280291/Put/seqid=0 2024-11-21T21:24:41,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742038_1214 (size=12983) 2024-11-21T21:24:41,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T21:24:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,644 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/93d5a2ab663f4fa9bff17d8cc7c41dd0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/93d5a2ab663f4fa9bff17d8cc7c41dd0 2024-11-21T21:24:41,649 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into 93d5a2ab663f4fa9bff17d8cc7c41dd0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:41,649 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:41,649 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=13, startTime=1732224281164; duration=0sec 2024-11-21T21:24:41,649 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:41,649 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:41,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224341660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224341660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224341669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224341669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/0534f5307967415ab09ea44133b8626a 2024-11-21T21:24:41,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/bed91d42e8e440f7ae92259356f10e85 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/bed91d42e8e440f7ae92259356f10e85 2024-11-21T21:24:41,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T21:24:41,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:41,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:41,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/bed91d42e8e440f7ae92259356f10e85, entries=250, sequenceid=299, filesize=47.5 K 2024-11-21T21:24:41,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/3db9359dff1c48ddac37f2172b01de54 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/3db9359dff1c48ddac37f2172b01de54 2024-11-21T21:24:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:24:41,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/3db9359dff1c48ddac37f2172b01de54, entries=150, sequenceid=299, filesize=12.0 K 2024-11-21T21:24:41,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/0534f5307967415ab09ea44133b8626a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0534f5307967415ab09ea44133b8626a 2024-11-21T21:24:41,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0534f5307967415ab09ea44133b8626a, entries=150, sequenceid=299, filesize=12.0 K 2024-11-21T21:24:41,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for cd5da8e0c287760dca4b3d875dd531a4 in 541ms, sequenceid=299, compaction requested=false 2024-11-21T21:24:41,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:41,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T21:24:41,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:41,944 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:24:41,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e66e9bd618f4cf9a4d65be8837b337f_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224281322/Put/seqid=0 2024-11-21T21:24:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:41,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:41,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224341982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224341984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224341985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224341986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:41,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742039_1215 (size=12454) 2024-11-21T21:24:41,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:41,997 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e66e9bd618f4cf9a4d65be8837b337f_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e66e9bd618f4cf9a4d65be8837b337f_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7922eba7546e4f83a200d9d0e3736fad, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7922eba7546e4f83a200d9d0e3736fad is 175, key is test_row_0/A:col10/1732224281322/Put/seqid=0 2024-11-21T21:24:42,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742040_1216 (size=31255) 2024-11-21T21:24:42,014 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7922eba7546e4f83a200d9d0e3736fad 2024-11-21T21:24:42,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/efdf4122597148d6a4f1cc9a9f7b144e is 50, key is test_row_0/B:col10/1732224281322/Put/seqid=0 2024-11-21T21:24:42,035 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/92ceb6089d0f45d19ebdb65ab3fb3c5c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/92ceb6089d0f45d19ebdb65ab3fb3c5c 2024-11-21T21:24:42,041 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into 92ceb6089d0f45d19ebdb65ab3fb3c5c(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:42,041 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:42,041 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=13, startTime=1732224281164; duration=0sec 2024-11-21T21:24:42,041 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:42,041 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:42,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742041_1217 (size=12301) 2024-11-21T21:24:42,053 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/efdf4122597148d6a4f1cc9a9f7b144e 2024-11-21T21:24:42,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a is 50, key is test_row_0/C:col10/1732224281322/Put/seqid=0 2024-11-21T21:24:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224342090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224342101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224342101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742042_1218 (size=12301) 2024-11-21T21:24:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224342102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,104 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a 2024-11-21T21:24:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7922eba7546e4f83a200d9d0e3736fad as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7922eba7546e4f83a200d9d0e3736fad 2024-11-21T21:24:42,116 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7922eba7546e4f83a200d9d0e3736fad, entries=150, sequenceid=326, filesize=30.5 K 2024-11-21T21:24:42,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/efdf4122597148d6a4f1cc9a9f7b144e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/efdf4122597148d6a4f1cc9a9f7b144e 2024-11-21T21:24:42,125 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/efdf4122597148d6a4f1cc9a9f7b144e, entries=150, sequenceid=326, filesize=12.0 K 2024-11-21T21:24:42,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T21:24:42,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a 2024-11-21T21:24:42,145 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-21T21:24:42,146 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for cd5da8e0c287760dca4b3d875dd531a4 in 202ms, sequenceid=326, compaction requested=true 2024-11-21T21:24:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T21:24:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T21:24:42,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T21:24:42,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 975 msec 2024-11-21T21:24:42,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 983 msec 2024-11-21T21:24:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:42,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:24:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:24:42,311 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:42,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T21:24:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:24:42,315 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:42,316 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:42,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:42,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211c4d9036f91c4e5abd370e61878d9f7f_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224282300/Put/seqid=0 2024-11-21T21:24:42,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742043_1219 (size=14994) 2024-11-21T21:24:42,349 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:42,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211c4d9036f91c4e5abd370e61878d9f7f_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211c4d9036f91c4e5abd370e61878d9f7f_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:42,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/c6a4b12c8107404fa37a1b2458f1cc49, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:42,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/c6a4b12c8107404fa37a1b2458f1cc49 is 175, key is test_row_0/A:col10/1732224282300/Put/seqid=0 2024-11-21T21:24:42,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224342360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224342361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224342361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224342362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742044_1220 (size=39949) 2024-11-21T21:24:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:24:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224342449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,454 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:24:42,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:42,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224342480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224342480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224342480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224342480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:24:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:24:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224342689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224342689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224342689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224342689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:42,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:24:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/c6a4b12c8107404fa37a1b2458f1cc49 2024-11-21T21:24:42,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/077e7f17e5ba44ec8afa12e0461e1f72 is 50, key is test_row_0/B:col10/1732224282300/Put/seqid=0 2024-11-21T21:24:42,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742045_1221 (size=12301) 2024-11-21T21:24:42,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/077e7f17e5ba44ec8afa12e0461e1f72 2024-11-21T21:24:42,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/627e55fc030845d7be8032530c17bb09 is 50, key is test_row_0/C:col10/1732224282300/Put/seqid=0 2024-11-21T21:24:42,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742046_1222 (size=12301) 2024-11-21T21:24:42,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:24:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:42,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:24:43,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224343002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224343002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224343002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224343004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:24:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:43,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:24:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:43,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/627e55fc030845d7be8032530c17bb09 2024-11-21T21:24:43,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/c6a4b12c8107404fa37a1b2458f1cc49 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/c6a4b12c8107404fa37a1b2458f1cc49 2024-11-21T21:24:43,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/c6a4b12c8107404fa37a1b2458f1cc49, entries=200, sequenceid=339, filesize=39.0 K 2024-11-21T21:24:43,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/077e7f17e5ba44ec8afa12e0461e1f72 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/077e7f17e5ba44ec8afa12e0461e1f72 2024-11-21T21:24:43,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/077e7f17e5ba44ec8afa12e0461e1f72, entries=150, sequenceid=339, filesize=12.0 K 2024-11-21T21:24:43,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/627e55fc030845d7be8032530c17bb09 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/627e55fc030845d7be8032530c17bb09 2024-11-21T21:24:43,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/627e55fc030845d7be8032530c17bb09, entries=150, sequenceid=339, filesize=12.0 K 2024-11-21T21:24:43,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cd5da8e0c287760dca4b3d875dd531a4 in 987ms, sequenceid=339, compaction requested=true 2024-11-21T21:24:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:43,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:43,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:43,292 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:43,292 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:43,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:43,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:43,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:43,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:43,293 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151780 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:43,293 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:43,293 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,293 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/93d5a2ab663f4fa9bff17d8cc7c41dd0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/bed91d42e8e440f7ae92259356f10e85, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7922eba7546e4f83a200d9d0e3736fad, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/c6a4b12c8107404fa37a1b2458f1cc49] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=148.2 K 2024-11-21T21:24:43,293 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:43,293 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,293 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:43,293 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/93d5a2ab663f4fa9bff17d8cc7c41dd0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/bed91d42e8e440f7ae92259356f10e85, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7922eba7546e4f83a200d9d0e3736fad, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/c6a4b12c8107404fa37a1b2458f1cc49] 2024-11-21T21:24:43,293 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,293 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/83bf59be1bc14a37bad6f4d30ed1942b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/3db9359dff1c48ddac37f2172b01de54, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/efdf4122597148d6a4f1cc9a9f7b144e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/077e7f17e5ba44ec8afa12e0461e1f72] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=48.7 K 2024-11-21T21:24:43,294 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d5a2ab663f4fa9bff17d8cc7c41dd0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732224280291 2024-11-21T21:24:43,294 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 83bf59be1bc14a37bad6f4d30ed1942b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732224280291 2024-11-21T21:24:43,294 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db9359dff1c48ddac37f2172b01de54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732224280658 2024-11-21T21:24:43,294 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bed91d42e8e440f7ae92259356f10e85, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732224280656 2024-11-21T21:24:43,294 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting efdf4122597148d6a4f1cc9a9f7b144e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732224281322 2024-11-21T21:24:43,294 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7922eba7546e4f83a200d9d0e3736fad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732224281322 2024-11-21T21:24:43,295 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 077e7f17e5ba44ec8afa12e0461e1f72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732224281981 2024-11-21T21:24:43,295 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a4b12c8107404fa37a1b2458f1cc49, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732224281981 2024-11-21T21:24:43,301 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:43,303 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:43,304 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/ee0c385d5dec4b9b867eb59843f6e803 is 50, key is test_row_0/B:col10/1732224282300/Put/seqid=0 2024-11-21T21:24:43,310 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d44585363b544678a69264804b3fa5f6_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:43,312 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d44585363b544678a69264804b3fa5f6_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:43,312 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d44585363b544678a69264804b3fa5f6_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:43,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742047_1223 (size=13119) 2024-11-21T21:24:43,321 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/ee0c385d5dec4b9b867eb59843f6e803 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ee0c385d5dec4b9b867eb59843f6e803 2024-11-21T21:24:43,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742048_1224 (size=4469) 2024-11-21T21:24:43,327 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into ee0c385d5dec4b9b867eb59843f6e803(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:43,327 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:43,327 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=12, startTime=1732224283292; duration=0sec 2024-11-21T21:24:43,327 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:43,327 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:43,327 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:43,328 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#183 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:43,328 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/1e48a0bddee34068b7d14a4a41244814 is 175, key is test_row_0/A:col10/1732224282300/Put/seqid=0 2024-11-21T21:24:43,328 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:43,328 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:43,328 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,328 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/92ceb6089d0f45d19ebdb65ab3fb3c5c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0534f5307967415ab09ea44133b8626a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/627e55fc030845d7be8032530c17bb09] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=48.7 K 2024-11-21T21:24:43,329 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ceb6089d0f45d19ebdb65ab3fb3c5c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732224280291 2024-11-21T21:24:43,329 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0534f5307967415ab09ea44133b8626a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732224280658 2024-11-21T21:24:43,330 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5fe5c648c2420e8fa8a0c3a8afcc0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732224281322 2024-11-21T21:24:43,330 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 627e55fc030845d7be8032530c17bb09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732224281981 2024-11-21T21:24:43,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742049_1225 (size=32073) 2024-11-21T21:24:43,341 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:43,341 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/acc8c6909bc8414e958387b0e26f174a is 50, key is test_row_0/C:col10/1732224282300/Put/seqid=0 2024-11-21T21:24:43,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742050_1226 (size=13119) 2024-11-21T21:24:43,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:24:43,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:43,395 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:24:43,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213983e5523e5c4f23a75ec6da5a713fbe_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224282339/Put/seqid=0 2024-11-21T21:24:43,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742051_1227 (size=12454) 2024-11-21T21:24:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:24:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:43,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224343520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224343520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224343520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224343524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224343630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224343630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224343639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,738 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/1e48a0bddee34068b7d14a4a41244814 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e48a0bddee34068b7d14a4a41244814 2024-11-21T21:24:43,742 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into 1e48a0bddee34068b7d14a4a41244814(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:43,742 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:43,742 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=12, startTime=1732224283291; duration=0sec 2024-11-21T21:24:43,742 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:43,742 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:43,761 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/acc8c6909bc8414e958387b0e26f174a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/acc8c6909bc8414e958387b0e26f174a 2024-11-21T21:24:43,767 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into acc8c6909bc8414e958387b0e26f174a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:43,767 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:43,767 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=12, startTime=1732224283292; duration=0sec 2024-11-21T21:24:43,768 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:43,768 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:43,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:43,819 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213983e5523e5c4f23a75ec6da5a713fbe_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213983e5523e5c4f23a75ec6da5a713fbe_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/08f72193067842848b30c5653c3bb3e9, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/08f72193067842848b30c5653c3bb3e9 is 175, key is test_row_0/A:col10/1732224282339/Put/seqid=0 2024-11-21T21:24:43,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742052_1228 (size=31255) 2024-11-21T21:24:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224343840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224343840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224343849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224344149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224344149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224344160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,225 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/08f72193067842848b30c5653c3bb3e9 2024-11-21T21:24:44,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/ba9b26275e13456e83b50af3a8e3eb28 is 50, key is test_row_0/B:col10/1732224282339/Put/seqid=0 2024-11-21T21:24:44,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742053_1229 (size=12301) 2024-11-21T21:24:44,244 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/ba9b26275e13456e83b50af3a8e3eb28 2024-11-21T21:24:44,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/adb07bde536943fb8ea5146150baa7a1 is 50, key is test_row_0/C:col10/1732224282339/Put/seqid=0 2024-11-21T21:24:44,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742054_1230 (size=12301) 2024-11-21T21:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:24:44,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224344540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:44,541 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224344660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,666 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/adb07bde536943fb8ea5146150baa7a1 2024-11-21T21:24:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/08f72193067842848b30c5653c3bb3e9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/08f72193067842848b30c5653c3bb3e9 2024-11-21T21:24:44,674 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/08f72193067842848b30c5653c3bb3e9, entries=150, sequenceid=364, filesize=30.5 K 2024-11-21T21:24:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/ba9b26275e13456e83b50af3a8e3eb28 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ba9b26275e13456e83b50af3a8e3eb28 2024-11-21T21:24:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224344670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224344670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:44,681 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ba9b26275e13456e83b50af3a8e3eb28, entries=150, sequenceid=364, filesize=12.0 K 2024-11-21T21:24:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/adb07bde536943fb8ea5146150baa7a1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/adb07bde536943fb8ea5146150baa7a1 2024-11-21T21:24:44,686 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/adb07bde536943fb8ea5146150baa7a1, entries=150, sequenceid=364, filesize=12.0 K 2024-11-21T21:24:44,687 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for cd5da8e0c287760dca4b3d875dd531a4 in 1292ms, sequenceid=364, compaction requested=false 2024-11-21T21:24:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T21:24:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T21:24:44,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T21:24:44,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3720 sec 2024-11-21T21:24:44,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.3800 sec 2024-11-21T21:24:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:45,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:45,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:24:45,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214e8365a1bfe34a7f841dee19eca83fc7_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224285671/Put/seqid=0 2024-11-21T21:24:45,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742055_1231 (size=14994) 2024-11-21T21:24:45,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:45,687 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214e8365a1bfe34a7f841dee19eca83fc7_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214e8365a1bfe34a7f841dee19eca83fc7_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:45,688 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/d32c0f6614c9417e901f4d64e0f12cc9, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:45,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/d32c0f6614c9417e901f4d64e0f12cc9 is 175, key is test_row_0/A:col10/1732224285671/Put/seqid=0 2024-11-21T21:24:45,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742056_1232 (size=39949) 2024-11-21T21:24:45,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/d32c0f6614c9417e901f4d64e0f12cc9 2024-11-21T21:24:45,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/77136c1e55e24cc691609dc5d6e61036 is 50, key is test_row_0/B:col10/1732224285671/Put/seqid=0 2024-11-21T21:24:45,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742057_1233 (size=12301) 2024-11-21T21:24:45,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/77136c1e55e24cc691609dc5d6e61036 2024-11-21T21:24:45,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/3da4f328b84043099dc9266cdc9c6b38 is 50, key is test_row_0/C:col10/1732224285671/Put/seqid=0 2024-11-21T21:24:45,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224345717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224345724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742058_1234 (size=12301) 2024-11-21T21:24:45,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/3da4f328b84043099dc9266cdc9c6b38 2024-11-21T21:24:45,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224345725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/d32c0f6614c9417e901f4d64e0f12cc9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d32c0f6614c9417e901f4d64e0f12cc9 2024-11-21T21:24:45,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d32c0f6614c9417e901f4d64e0f12cc9, entries=200, sequenceid=379, filesize=39.0 K 2024-11-21T21:24:45,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/77136c1e55e24cc691609dc5d6e61036 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/77136c1e55e24cc691609dc5d6e61036 2024-11-21T21:24:45,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/77136c1e55e24cc691609dc5d6e61036, entries=150, sequenceid=379, filesize=12.0 K 2024-11-21T21:24:45,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/3da4f328b84043099dc9266cdc9c6b38 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3da4f328b84043099dc9266cdc9c6b38 2024-11-21T21:24:45,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3da4f328b84043099dc9266cdc9c6b38, entries=150, sequenceid=379, filesize=12.0 K 2024-11-21T21:24:45,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cd5da8e0c287760dca4b3d875dd531a4 in 81ms, sequenceid=379, compaction requested=true 2024-11-21T21:24:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:45,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:45,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:45,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:45,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:45,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:45,754 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:45,755 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:45,755 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:45,755 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ee0c385d5dec4b9b867eb59843f6e803, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ba9b26275e13456e83b50af3a8e3eb28, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/77136c1e55e24cc691609dc5d6e61036] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=36.8 K 2024-11-21T21:24:45,755 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e48a0bddee34068b7d14a4a41244814, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/08f72193067842848b30c5653c3bb3e9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d32c0f6614c9417e901f4d64e0f12cc9] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=100.9 K 2024-11-21T21:24:45,755 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:45,755 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e48a0bddee34068b7d14a4a41244814, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/08f72193067842848b30c5653c3bb3e9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d32c0f6614c9417e901f4d64e0f12cc9] 2024-11-21T21:24:45,756 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e48a0bddee34068b7d14a4a41244814, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732224281981 2024-11-21T21:24:45,756 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ee0c385d5dec4b9b867eb59843f6e803, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732224281981 2024-11-21T21:24:45,757 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08f72193067842848b30c5653c3bb3e9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732224282339 2024-11-21T21:24:45,757 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9b26275e13456e83b50af3a8e3eb28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732224282339 2024-11-21T21:24:45,757 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32c0f6614c9417e901f4d64e0f12cc9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732224283517 2024-11-21T21:24:45,757 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 77136c1e55e24cc691609dc5d6e61036, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732224283517 2024-11-21T21:24:45,764 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:45,765 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:45,765 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/79a16053880e404e8ae2a7019271ab4e is 50, key is test_row_0/B:col10/1732224285671/Put/seqid=0 2024-11-21T21:24:45,767 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d56578df327145bbb1a10e5733a83cc8_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:45,769 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d56578df327145bbb1a10e5733a83cc8_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:45,769 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d56578df327145bbb1a10e5733a83cc8_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:45,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742059_1235 (size=13221) 2024-11-21T21:24:45,780 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/79a16053880e404e8ae2a7019271ab4e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/79a16053880e404e8ae2a7019271ab4e 2024-11-21T21:24:45,784 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into 79a16053880e404e8ae2a7019271ab4e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:45,784 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:45,784 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=13, startTime=1732224285754; duration=0sec 2024-11-21T21:24:45,785 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:45,785 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:45,785 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:24:45,787 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:24:45,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742060_1236 (size=4469) 2024-11-21T21:24:45,787 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:45,787 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:45,788 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/acc8c6909bc8414e958387b0e26f174a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/adb07bde536943fb8ea5146150baa7a1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3da4f328b84043099dc9266cdc9c6b38] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=36.8 K 2024-11-21T21:24:45,788 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting acc8c6909bc8414e958387b0e26f174a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732224281981 2024-11-21T21:24:45,788 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting adb07bde536943fb8ea5146150baa7a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732224282339 2024-11-21T21:24:45,789 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da4f328b84043099dc9266cdc9c6b38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732224283517 2024-11-21T21:24:45,796 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:45,796 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/01e66afdccc94b358b11b3f3bdf852f4 is 50, key is test_row_0/C:col10/1732224285671/Put/seqid=0 2024-11-21T21:24:45,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742061_1237 (size=13221) 2024-11-21T21:24:45,805 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/01e66afdccc94b358b11b3f3bdf852f4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/01e66afdccc94b358b11b3f3bdf852f4 2024-11-21T21:24:45,813 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into 01e66afdccc94b358b11b3f3bdf852f4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:45,813 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:45,813 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=13, startTime=1732224285754; duration=0sec 2024-11-21T21:24:45,813 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:45,813 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:45,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:24:45,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e4544e2a9bad46f08a5eacee35ee98f4_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224285723/Put/seqid=0 2024-11-21T21:24:45,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742062_1238 (size=12454) 2024-11-21T21:24:45,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224345851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224345852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224345855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224345959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224345959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224345959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224346169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224346169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224346169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,189 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#193 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:46,189 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/41c7a75fb2aa4ccdb86114c5046829d3 is 175, key is test_row_0/A:col10/1732224285671/Put/seqid=0 2024-11-21T21:24:46,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742063_1239 (size=32175) 2024-11-21T21:24:46,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:46,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e4544e2a9bad46f08a5eacee35ee98f4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4544e2a9bad46f08a5eacee35ee98f4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:46,268 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/e7299a66620647c79887896adf1941b4, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:46,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/e7299a66620647c79887896adf1941b4 is 175, key is test_row_0/A:col10/1732224285723/Put/seqid=0 2024-11-21T21:24:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742064_1240 (size=31255) 2024-11-21T21:24:46,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/e7299a66620647c79887896adf1941b4 2024-11-21T21:24:46,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/fb3a4f94d4ef47eeb1770fd2439f26d9 is 50, key is test_row_0/B:col10/1732224285723/Put/seqid=0 2024-11-21T21:24:46,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742065_1241 (size=12301) 2024-11-21T21:24:46,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/fb3a4f94d4ef47eeb1770fd2439f26d9 2024-11-21T21:24:46,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/f59ec9e4970b4847b530e503c0637a9b is 50, key is test_row_0/C:col10/1732224285723/Put/seqid=0 2024-11-21T21:24:46,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742066_1242 (size=12301) 2024-11-21T21:24:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:24:46,480 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:46,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,481 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:24:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53602 deadline: 1732224346480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224346480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,481 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T21:24:46,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224346480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:24:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224346480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:24:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,484 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:24:46,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:24:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53606 deadline: 1732224346559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:46,561 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in cd5da8e0c287760dca4b3d875dd531a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:24:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:24:46,598 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/41c7a75fb2aa4ccdb86114c5046829d3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/41c7a75fb2aa4ccdb86114c5046829d3 2024-11-21T21:24:46,602 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into 41c7a75fb2aa4ccdb86114c5046829d3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:46,602 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:46,602 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=13, startTime=1732224285753; duration=0sec 2024-11-21T21:24:46,602 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:46,602 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:46,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T21:24:46,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:46,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:46,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:46,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:46,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:24:46,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/f59ec9e4970b4847b530e503c0637a9b 2024-11-21T21:24:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/e7299a66620647c79887896adf1941b4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e7299a66620647c79887896adf1941b4 2024-11-21T21:24:46,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e7299a66620647c79887896adf1941b4, entries=150, sequenceid=405, filesize=30.5 K 2024-11-21T21:24:46,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/fb3a4f94d4ef47eeb1770fd2439f26d9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fb3a4f94d4ef47eeb1770fd2439f26d9 2024-11-21T21:24:46,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fb3a4f94d4ef47eeb1770fd2439f26d9, entries=150, sequenceid=405, filesize=12.0 K 2024-11-21T21:24:46,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/f59ec9e4970b4847b530e503c0637a9b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f59ec9e4970b4847b530e503c0637a9b 2024-11-21T21:24:46,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f59ec9e4970b4847b530e503c0637a9b, entries=150, sequenceid=405, filesize=12.0 K 2024-11-21T21:24:46,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for cd5da8e0c287760dca4b3d875dd531a4 in 891ms, sequenceid=405, compaction requested=false 2024-11-21T21:24:46,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:46,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T21:24:46,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:46,789 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:24:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121364cfe5ff5ca4ff087854e96cc16b00b_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224285850/Put/seqid=0 2024-11-21T21:24:46,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742067_1243 (size=12454) 2024-11-21T21:24:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:24:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:46,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. as already flushing 2024-11-21T21:24:47,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224347013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224347014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224347015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:24:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224347130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224347130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224347130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,139 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:24:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:47,202 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121364cfe5ff5ca4ff087854e96cc16b00b_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121364cfe5ff5ca4ff087854e96cc16b00b_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:47,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b583aeb65eb048d583a5c49a1cfaacd6, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:47,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b583aeb65eb048d583a5c49a1cfaacd6 is 175, key is test_row_0/A:col10/1732224285850/Put/seqid=0 2024-11-21T21:24:47,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742068_1244 (size=31255) 2024-11-21T21:24:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224347339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224347339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224347339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,607 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b583aeb65eb048d583a5c49a1cfaacd6 2024-11-21T21:24:47,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/6628524c103c435c8db0fbe886d1fb07 is 50, key is test_row_0/B:col10/1732224285850/Put/seqid=0 2024-11-21T21:24:47,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742069_1245 (size=12301) 2024-11-21T21:24:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:24:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224347650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224347650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224347650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:48,018 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/6628524c103c435c8db0fbe886d1fb07 2024-11-21T21:24:48,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e is 50, key is test_row_0/C:col10/1732224285850/Put/seqid=0 2024-11-21T21:24:48,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742070_1246 (size=12301) 2024-11-21T21:24:48,033 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e 2024-11-21T21:24:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/b583aeb65eb048d583a5c49a1cfaacd6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b583aeb65eb048d583a5c49a1cfaacd6 2024-11-21T21:24:48,051 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b583aeb65eb048d583a5c49a1cfaacd6, entries=150, sequenceid=418, filesize=30.5 K 2024-11-21T21:24:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/6628524c103c435c8db0fbe886d1fb07 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6628524c103c435c8db0fbe886d1fb07 2024-11-21T21:24:48,058 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6628524c103c435c8db0fbe886d1fb07, entries=150, sequenceid=418, filesize=12.0 K 2024-11-21T21:24:48,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e 2024-11-21T21:24:48,069 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e, entries=150, sequenceid=418, filesize=12.0 K 2024-11-21T21:24:48,070 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cd5da8e0c287760dca4b3d875dd531a4 in 1281ms, sequenceid=418, compaction requested=true 2024-11-21T21:24:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T21:24:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T21:24:48,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T21:24:48,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5880 sec 2024-11-21T21:24:48,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5950 sec 2024-11-21T21:24:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:48,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:24:48,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:48,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224348172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224348173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:48,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cacf31fd7ee34eecb8d91f4cd7dbe7bc_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224288160/Put/seqid=0 2024-11-21T21:24:48,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224348175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:48,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742071_1247 (size=12454) 2024-11-21T21:24:48,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:48,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cacf31fd7ee34eecb8d91f4cd7dbe7bc_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cacf31fd7ee34eecb8d91f4cd7dbe7bc_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:48,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7c7e3ab2fec64433a3d04a352d2f8598, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:48,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7c7e3ab2fec64433a3d04a352d2f8598 is 175, key is test_row_0/A:col10/1732224288160/Put/seqid=0 2024-11-21T21:24:48,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742072_1248 (size=31255) 2024-11-21T21:24:48,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7c7e3ab2fec64433a3d04a352d2f8598 2024-11-21T21:24:48,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2393216d42be4626804e634bfe6be1c6 is 50, key is test_row_0/B:col10/1732224288160/Put/seqid=0 2024-11-21T21:24:48,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742073_1249 (size=12301) 2024-11-21T21:24:48,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2393216d42be4626804e634bfe6be1c6 2024-11-21T21:24:48,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/9b281355033d45b09219f812f83c4953 is 50, key is test_row_0/C:col10/1732224288160/Put/seqid=0 2024-11-21T21:24:48,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742074_1250 (size=12301) 2024-11-21T21:24:48,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/9b281355033d45b09219f812f83c4953 2024-11-21T21:24:48,274 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-21T21:24:48,274 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:48,274 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,274 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:48,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/7c7e3ab2fec64433a3d04a352d2f8598 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7c7e3ab2fec64433a3d04a352d2f8598 2024-11-21T21:24:48,277 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-21T21:24:48,277 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-21T21:24:48,277 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:48,277 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:48,277 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,277 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,277 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,277 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:48,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:48,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7c7e3ab2fec64433a3d04a352d2f8598, entries=150, sequenceid=443, filesize=30.5 K 2024-11-21T21:24:48,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/2393216d42be4626804e634bfe6be1c6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2393216d42be4626804e634bfe6be1c6 2024-11-21T21:24:48,283 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-21T21:24:48,283 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:48,283 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,283 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:48,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2393216d42be4626804e634bfe6be1c6, entries=150, sequenceid=443, filesize=12.0 K 2024-11-21T21:24:48,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/9b281355033d45b09219f812f83c4953 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9b281355033d45b09219f812f83c4953 2024-11-21T21:24:48,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:48,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53548 deadline: 1732224348289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53594 deadline: 1732224348289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:48,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:24:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732224348289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 2024-11-21T21:24:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:24:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:24:48,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9b281355033d45b09219f812f83c4953, entries=150, sequenceid=443, filesize=12.0 K 2024-11-21T21:24:48,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for cd5da8e0c287760dca4b3d875dd531a4 in 131ms, sequenceid=443, compaction requested=true 2024-11-21T21:24:48,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:48,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:24:48,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:48,293 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:48,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:24:48,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:48,294 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:48,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cd5da8e0c287760dca4b3d875dd531a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:24:48,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125940 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/B is initiating minor compaction (all files) 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/A is initiating minor compaction (all files) 2024-11-21T21:24:48,295 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/B in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:48,295 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/A in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:48,295 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/41c7a75fb2aa4ccdb86114c5046829d3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e7299a66620647c79887896adf1941b4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b583aeb65eb048d583a5c49a1cfaacd6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7c7e3ab2fec64433a3d04a352d2f8598] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=123.0 K 2024-11-21T21:24:48,295 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/79a16053880e404e8ae2a7019271ab4e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fb3a4f94d4ef47eeb1770fd2439f26d9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6628524c103c435c8db0fbe886d1fb07, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2393216d42be4626804e634bfe6be1c6] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=48.9 K 2024-11-21T21:24:48,295 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/41c7a75fb2aa4ccdb86114c5046829d3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e7299a66620647c79887896adf1941b4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b583aeb65eb048d583a5c49a1cfaacd6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7c7e3ab2fec64433a3d04a352d2f8598] 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a16053880e404e8ae2a7019271ab4e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732224283517 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c7a75fb2aa4ccdb86114c5046829d3, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732224283517 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting fb3a4f94d4ef47eeb1770fd2439f26d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732224285721 2024-11-21T21:24:48,295 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7299a66620647c79887896adf1941b4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732224285721 2024-11-21T21:24:48,296 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6628524c103c435c8db0fbe886d1fb07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732224285837 2024-11-21T21:24:48,296 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b583aeb65eb048d583a5c49a1cfaacd6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732224285837 2024-11-21T21:24:48,296 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2393216d42be4626804e634bfe6be1c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732224287013 2024-11-21T21:24:48,296 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7e3ab2fec64433a3d04a352d2f8598, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732224287013 2024-11-21T21:24:48,309 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:48,310 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#B#compaction#204 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:48,310 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/010d4ee569ed4827b7ad187d0d594806 is 50, key is test_row_0/B:col10/1732224288160/Put/seqid=0 2024-11-21T21:24:48,312 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210fd215e33ade43f5a628418181cbee51_cd5da8e0c287760dca4b3d875dd531a4 store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:48,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742075_1251 (size=13357) 2024-11-21T21:24:48,333 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/010d4ee569ed4827b7ad187d0d594806 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/010d4ee569ed4827b7ad187d0d594806 2024-11-21T21:24:48,339 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/B of cd5da8e0c287760dca4b3d875dd531a4 into 010d4ee569ed4827b7ad187d0d594806(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:48,339 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:48,339 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/B, priority=12, startTime=1732224288293; duration=0sec 2024-11-21T21:24:48,339 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:24:48,340 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:B 2024-11-21T21:24:48,340 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:24:48,341 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:24:48,342 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): cd5da8e0c287760dca4b3d875dd531a4/C is initiating minor compaction (all files) 2024-11-21T21:24:48,342 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cd5da8e0c287760dca4b3d875dd531a4/C in TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:48,342 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/01e66afdccc94b358b11b3f3bdf852f4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f59ec9e4970b4847b530e503c0637a9b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9b281355033d45b09219f812f83c4953] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp, totalSize=48.9 K 2024-11-21T21:24:48,342 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e66afdccc94b358b11b3f3bdf852f4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732224283517 2024-11-21T21:24:48,343 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f59ec9e4970b4847b530e503c0637a9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732224285721 2024-11-21T21:24:48,343 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b2d669a78ef84f1dbbcbbcfcb2e27c9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732224285837 2024-11-21T21:24:48,343 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b281355033d45b09219f812f83c4953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732224287013 2024-11-21T21:24:48,344 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210fd215e33ade43f5a628418181cbee51_cd5da8e0c287760dca4b3d875dd531a4, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:48,345 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210fd215e33ade43f5a628418181cbee51_cd5da8e0c287760dca4b3d875dd531a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:48,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742076_1252 (size=4469) 2024-11-21T21:24:48,351 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#A#compaction#205 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:48,352 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/d8b1beeeaed84130bd6f982a6575649a is 175, key is test_row_0/A:col10/1732224288160/Put/seqid=0 2024-11-21T21:24:48,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742077_1253 (size=32311) 2024-11-21T21:24:48,361 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cd5da8e0c287760dca4b3d875dd531a4#C#compaction#206 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:24:48,361 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/25c6704545424224b2c0fd9082a7aea1 is 50, key is test_row_0/C:col10/1732224288160/Put/seqid=0 2024-11-21T21:24:48,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742078_1254 (size=13357) 2024-11-21T21:24:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:48,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:24:48,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:24:48,501 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-21T21:24:48,501 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:48,501 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,502 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-21T21:24:48,502 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-21T21:24:48,502 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:48,502 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,502 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,502 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:48,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:48,502 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,502 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,502 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:48,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:48,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:48,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112149c43d9b6e6c4c9cad09a3ce8fac9d21_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_0/A:col10/1732224288174/Put/seqid=0 2024-11-21T21:24:48,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742079_1255 (size=12454) 2024-11-21T21:24:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:24:48,630 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:48,770 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/d8b1beeeaed84130bd6f982a6575649a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d8b1beeeaed84130bd6f982a6575649a 2024-11-21T21:24:48,771 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/25c6704545424224b2c0fd9082a7aea1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/25c6704545424224b2c0fd9082a7aea1 2024-11-21T21:24:48,774 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/A of cd5da8e0c287760dca4b3d875dd531a4 into d8b1beeeaed84130bd6f982a6575649a(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:48,774 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:48,774 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/A, priority=12, startTime=1732224288293; duration=0sec 2024-11-21T21:24:48,774 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:48,774 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:A 2024-11-21T21:24:48,775 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cd5da8e0c287760dca4b3d875dd531a4/C of cd5da8e0c287760dca4b3d875dd531a4 into 25c6704545424224b2c0fd9082a7aea1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:24:48,775 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:48,775 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., storeName=cd5da8e0c287760dca4b3d875dd531a4/C, priority=12, startTime=1732224288294; duration=0sec 2024-11-21T21:24:48,775 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:24:48,775 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cd5da8e0c287760dca4b3d875dd531a4:C 2024-11-21T21:24:48,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:48,917 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112149c43d9b6e6c4c9cad09a3ce8fac9d21_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112149c43d9b6e6c4c9cad09a3ce8fac9d21_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:48,918 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/6b7c8463c90c4650a3332d484f2711c4, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:48,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/6b7c8463c90c4650a3332d484f2711c4 is 175, key is test_row_0/A:col10/1732224288174/Put/seqid=0 2024-11-21T21:24:48,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742080_1256 (size=31255) 2024-11-21T21:24:49,325 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/6b7c8463c90c4650a3332d484f2711c4 2024-11-21T21:24:49,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/dd9a95ba7ba1454e807ac57d707fde52 is 50, key is test_row_0/B:col10/1732224288174/Put/seqid=0 2024-11-21T21:24:49,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742081_1257 (size=12301) 2024-11-21T21:24:49,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/dd9a95ba7ba1454e807ac57d707fde52 2024-11-21T21:24:49,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/dbff31937bc542feaf35bf7178a46ecc is 50, key is test_row_0/C:col10/1732224288174/Put/seqid=0 2024-11-21T21:24:49,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742082_1258 (size=12301) 2024-11-21T21:24:49,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/dbff31937bc542feaf35bf7178a46ecc 2024-11-21T21:24:49,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/6b7c8463c90c4650a3332d484f2711c4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/6b7c8463c90c4650a3332d484f2711c4 2024-11-21T21:24:49,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/6b7c8463c90c4650a3332d484f2711c4, entries=150, sequenceid=458, filesize=30.5 K 2024-11-21T21:24:49,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/dd9a95ba7ba1454e807ac57d707fde52 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/dd9a95ba7ba1454e807ac57d707fde52 2024-11-21T21:24:49,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/dd9a95ba7ba1454e807ac57d707fde52, entries=150, sequenceid=458, filesize=12.0 K 2024-11-21T21:24:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/dbff31937bc542feaf35bf7178a46ecc as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/dbff31937bc542feaf35bf7178a46ecc 2024-11-21T21:24:49,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/dbff31937bc542feaf35bf7178a46ecc, entries=150, sequenceid=458, filesize=12.0 K 2024-11-21T21:24:49,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for cd5da8e0c287760dca4b3d875dd531a4 in 1287ms, sequenceid=458, compaction requested=false 2024-11-21T21:24:49,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cd5da8e0c287760dca4b3d875dd531a4: 2024-11-21T21:24:50,601 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-21T21:24:50,601 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:50,601 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:50,601 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:50,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:56,501 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-21T21:24:56,501 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:24:56,501 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:56,501 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:56,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3481 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3380 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1488 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4464 rows 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1490 2024-11-21T21:24:56,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4452 rows 2024-11-21T21:24:56,501 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:24:56,502 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:24:56,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:56,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:24:56,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:24:56,503 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:24:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:24:56,506 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224296506"}]},"ts":"1732224296506"} 2024-11-21T21:24:56,508 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:24:56,508 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:24:56,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:24:56,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, UNASSIGN}] 2024-11-21T21:24:56,511 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, UNASSIGN 2024-11-21T21:24:56,512 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:56,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, UNASSIGN because future has completed 2024-11-21T21:24:56,514 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:24:56,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:24:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:24:56,667 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:56,667 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:24:56,668 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing cd5da8e0c287760dca4b3d875dd531a4, disabling compactions & flushes 2024-11-21T21:24:56,668 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:56,668 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:56,668 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. after waiting 0 ms 2024-11-21T21:24:56,668 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:56,668 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing cd5da8e0c287760dca4b3d875dd531a4 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-21T21:24:56,676 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e17ff83db12c4044aa32aaa3205bef83_cd5da8e0c287760dca4b3d875dd531a4 is 50, key is test_row_2/A:col10/1732224296500/Put/seqid=0 2024-11-21T21:24:56,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742083_1259 (size=7374) 2024-11-21T21:24:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:24:57,084 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:57,089 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e17ff83db12c4044aa32aaa3205bef83_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e17ff83db12c4044aa32aaa3205bef83_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:57,090 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/861159a9f3b54d849a38a46393a7c62b, store: [table=TestAcidGuarantees family=A region=cd5da8e0c287760dca4b3d875dd531a4] 2024-11-21T21:24:57,091 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/861159a9f3b54d849a38a46393a7c62b is 175, key is test_row_2/A:col10/1732224296500/Put/seqid=0 2024-11-21T21:24:57,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742084_1260 (size=13865) 2024-11-21T21:24:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:24:57,496 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=465, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/861159a9f3b54d849a38a46393a7c62b 2024-11-21T21:24:57,502 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/7a4a7b2e17ff4eb1911659c1171fc283 is 50, key is test_row_2/B:col10/1732224296500/Put/seqid=0 2024-11-21T21:24:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742085_1261 (size=7415) 2024-11-21T21:24:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:24:57,910 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/7a4a7b2e17ff4eb1911659c1171fc283 2024-11-21T21:24:57,917 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/7f9d86ac1fcb43fda8b0f1bb525159fd is 50, key is test_row_2/C:col10/1732224296500/Put/seqid=0 2024-11-21T21:24:57,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742086_1262 (size=7415) 2024-11-21T21:24:58,321 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/7f9d86ac1fcb43fda8b0f1bb525159fd 2024-11-21T21:24:58,325 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/A/861159a9f3b54d849a38a46393a7c62b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/861159a9f3b54d849a38a46393a7c62b 2024-11-21T21:24:58,329 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/861159a9f3b54d849a38a46393a7c62b, entries=50, sequenceid=465, filesize=13.5 K 2024-11-21T21:24:58,330 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/B/7a4a7b2e17ff4eb1911659c1171fc283 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/7a4a7b2e17ff4eb1911659c1171fc283 2024-11-21T21:24:58,333 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/7a4a7b2e17ff4eb1911659c1171fc283, entries=50, sequenceid=465, filesize=7.2 K 2024-11-21T21:24:58,334 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/.tmp/C/7f9d86ac1fcb43fda8b0f1bb525159fd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/7f9d86ac1fcb43fda8b0f1bb525159fd 2024-11-21T21:24:58,338 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/7f9d86ac1fcb43fda8b0f1bb525159fd, entries=50, sequenceid=465, filesize=7.2 K 2024-11-21T21:24:58,339 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for cd5da8e0c287760dca4b3d875dd531a4 in 1671ms, sequenceid=465, compaction requested=true 2024-11-21T21:24:58,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/eff7e45c122f4659a9b8a75ee1f8f559, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e758a120ec24bd1b3a5b0161d887eb4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/464b9a6a313545d5bfd9f246b59d84ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/877ecdee0790447d844a87e097338941, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e096796f9bf44eb8b9d63973466b5005, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/accf3e4adbb3409d8b20b3fcf40751b8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/62f56a81f13549a0bed3b1fa1be0d4d5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0d9095df52d74a0ab55429ed167b7c14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/9d3569fe2f8e48df92d6eb65b636c7c6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0587a2ae33424d67a2a0a0c2d2555c10, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/333f67fc99314262af9601fa73b2fbdf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/05ecce48816f46c49bcd295d868fede1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/468b757ee4e242c4b0dbfd4e1a66b265, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7edfd81747c743e994de4c1c648926b2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/02389cf6249248f6a9ea17b5738dbd59, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7193820220de4f098c575f9e0fa3fd76, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/4b23457c6a084d349b298272cc5b5125, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/48a317a02d204d19977f80c5214562a4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/f71d151386164e3f95bf922074843587, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/93d5a2ab663f4fa9bff17d8cc7c41dd0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b836f49f6e5f42a1afaa9d3a2a1f4d66, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/bed91d42e8e440f7ae92259356f10e85, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7922eba7546e4f83a200d9d0e3736fad, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/c6a4b12c8107404fa37a1b2458f1cc49, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e48a0bddee34068b7d14a4a41244814, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/08f72193067842848b30c5653c3bb3e9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d32c0f6614c9417e901f4d64e0f12cc9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/41c7a75fb2aa4ccdb86114c5046829d3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e7299a66620647c79887896adf1941b4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b583aeb65eb048d583a5c49a1cfaacd6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7c7e3ab2fec64433a3d04a352d2f8598] to archive 2024-11-21T21:24:58,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:24:58,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/eff7e45c122f4659a9b8a75ee1f8f559 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/eff7e45c122f4659a9b8a75ee1f8f559 2024-11-21T21:24:58,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e758a120ec24bd1b3a5b0161d887eb4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e758a120ec24bd1b3a5b0161d887eb4 2024-11-21T21:24:58,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/464b9a6a313545d5bfd9f246b59d84ff to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/464b9a6a313545d5bfd9f246b59d84ff 2024-11-21T21:24:58,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/877ecdee0790447d844a87e097338941 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/877ecdee0790447d844a87e097338941 2024-11-21T21:24:58,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e096796f9bf44eb8b9d63973466b5005 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e096796f9bf44eb8b9d63973466b5005 2024-11-21T21:24:58,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/accf3e4adbb3409d8b20b3fcf40751b8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/accf3e4adbb3409d8b20b3fcf40751b8 2024-11-21T21:24:58,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/62f56a81f13549a0bed3b1fa1be0d4d5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/62f56a81f13549a0bed3b1fa1be0d4d5 2024-11-21T21:24:58,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0d9095df52d74a0ab55429ed167b7c14 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0d9095df52d74a0ab55429ed167b7c14 2024-11-21T21:24:58,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/9d3569fe2f8e48df92d6eb65b636c7c6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/9d3569fe2f8e48df92d6eb65b636c7c6 2024-11-21T21:24:58,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0587a2ae33424d67a2a0a0c2d2555c10 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/0587a2ae33424d67a2a0a0c2d2555c10 2024-11-21T21:24:58,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/333f67fc99314262af9601fa73b2fbdf to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/333f67fc99314262af9601fa73b2fbdf 2024-11-21T21:24:58,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/05ecce48816f46c49bcd295d868fede1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/05ecce48816f46c49bcd295d868fede1 2024-11-21T21:24:58,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/468b757ee4e242c4b0dbfd4e1a66b265 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/468b757ee4e242c4b0dbfd4e1a66b265 2024-11-21T21:24:58,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7edfd81747c743e994de4c1c648926b2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7edfd81747c743e994de4c1c648926b2 2024-11-21T21:24:58,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/02389cf6249248f6a9ea17b5738dbd59 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/02389cf6249248f6a9ea17b5738dbd59 2024-11-21T21:24:58,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7193820220de4f098c575f9e0fa3fd76 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7193820220de4f098c575f9e0fa3fd76 2024-11-21T21:24:58,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/4b23457c6a084d349b298272cc5b5125 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/4b23457c6a084d349b298272cc5b5125 2024-11-21T21:24:58,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/48a317a02d204d19977f80c5214562a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/48a317a02d204d19977f80c5214562a4 2024-11-21T21:24:58,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/f71d151386164e3f95bf922074843587 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/f71d151386164e3f95bf922074843587 2024-11-21T21:24:58,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/93d5a2ab663f4fa9bff17d8cc7c41dd0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/93d5a2ab663f4fa9bff17d8cc7c41dd0 2024-11-21T21:24:58,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b836f49f6e5f42a1afaa9d3a2a1f4d66 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b836f49f6e5f42a1afaa9d3a2a1f4d66 2024-11-21T21:24:58,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/bed91d42e8e440f7ae92259356f10e85 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/bed91d42e8e440f7ae92259356f10e85 2024-11-21T21:24:58,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7922eba7546e4f83a200d9d0e3736fad to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7922eba7546e4f83a200d9d0e3736fad 2024-11-21T21:24:58,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/c6a4b12c8107404fa37a1b2458f1cc49 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/c6a4b12c8107404fa37a1b2458f1cc49 2024-11-21T21:24:58,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e48a0bddee34068b7d14a4a41244814 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/1e48a0bddee34068b7d14a4a41244814 2024-11-21T21:24:58,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/08f72193067842848b30c5653c3bb3e9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/08f72193067842848b30c5653c3bb3e9 2024-11-21T21:24:58,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d32c0f6614c9417e901f4d64e0f12cc9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d32c0f6614c9417e901f4d64e0f12cc9 2024-11-21T21:24:58,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/41c7a75fb2aa4ccdb86114c5046829d3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/41c7a75fb2aa4ccdb86114c5046829d3 2024-11-21T21:24:58,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e7299a66620647c79887896adf1941b4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/e7299a66620647c79887896adf1941b4 2024-11-21T21:24:58,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b583aeb65eb048d583a5c49a1cfaacd6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/b583aeb65eb048d583a5c49a1cfaacd6 2024-11-21T21:24:58,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7c7e3ab2fec64433a3d04a352d2f8598 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/7c7e3ab2fec64433a3d04a352d2f8598 2024-11-21T21:24:58,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/450aa89574c3469fbe39e0f079664a0d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fbd99840c4aa42e0986c0fbae1d74fa9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a898a3db25114b60b0c687c47fa0f7b2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a17cc3c19de54469b0edcd4a5997a284, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a3b54c5747594abb84561e2fec1c23fe, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6ad341fe5a3b4e468cbafe4fd5efc3a1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f8c96d84f24f4d54941ecdbcde912cea, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/963eb19968d047e3a932f510fad20feb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb47067ce7294763a9ede23c2dbdc2e7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/090c6187b42f48ea943802eb729daef7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/88aeab420fd9415bb05a238e07c46bfd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/578e748ca8b94701b85b2a0b32f19de0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/d6db316d639e4a5e8451a8ff04b89616, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/06ca25c7e4644bcf91c7757cd8e7235b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2356493c88b24db999ebd8924a4bbbaf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2f2a6c5e670e46ccb98ad3c9e934e753, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/8434250f9d0f4735bdc5206eb6a315d8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/1e83c0c0a0fb4c6fb2efe9780a518e8c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f2a18c1801054380aa68e59b60a45193, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/83bf59be1bc14a37bad6f4d30ed1942b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb75cacdf04e4af9b58b3880df513c39, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/3db9359dff1c48ddac37f2172b01de54, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/efdf4122597148d6a4f1cc9a9f7b144e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ee0c385d5dec4b9b867eb59843f6e803, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/077e7f17e5ba44ec8afa12e0461e1f72, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ba9b26275e13456e83b50af3a8e3eb28, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/79a16053880e404e8ae2a7019271ab4e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/77136c1e55e24cc691609dc5d6e61036, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fb3a4f94d4ef47eeb1770fd2439f26d9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6628524c103c435c8db0fbe886d1fb07, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2393216d42be4626804e634bfe6be1c6] to archive 2024-11-21T21:24:58,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:24:58,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/450aa89574c3469fbe39e0f079664a0d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/450aa89574c3469fbe39e0f079664a0d 2024-11-21T21:24:58,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fbd99840c4aa42e0986c0fbae1d74fa9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fbd99840c4aa42e0986c0fbae1d74fa9 2024-11-21T21:24:58,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a898a3db25114b60b0c687c47fa0f7b2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a898a3db25114b60b0c687c47fa0f7b2 2024-11-21T21:24:58,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a17cc3c19de54469b0edcd4a5997a284 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a17cc3c19de54469b0edcd4a5997a284 2024-11-21T21:24:58,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a3b54c5747594abb84561e2fec1c23fe to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/a3b54c5747594abb84561e2fec1c23fe 2024-11-21T21:24:58,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6ad341fe5a3b4e468cbafe4fd5efc3a1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6ad341fe5a3b4e468cbafe4fd5efc3a1 2024-11-21T21:24:58,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f8c96d84f24f4d54941ecdbcde912cea to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f8c96d84f24f4d54941ecdbcde912cea 2024-11-21T21:24:58,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/963eb19968d047e3a932f510fad20feb to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/963eb19968d047e3a932f510fad20feb 2024-11-21T21:24:58,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb47067ce7294763a9ede23c2dbdc2e7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb47067ce7294763a9ede23c2dbdc2e7 2024-11-21T21:24:58,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/090c6187b42f48ea943802eb729daef7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/090c6187b42f48ea943802eb729daef7 2024-11-21T21:24:58,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/88aeab420fd9415bb05a238e07c46bfd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/88aeab420fd9415bb05a238e07c46bfd 2024-11-21T21:24:58,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/578e748ca8b94701b85b2a0b32f19de0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/578e748ca8b94701b85b2a0b32f19de0 2024-11-21T21:24:58,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/d6db316d639e4a5e8451a8ff04b89616 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/d6db316d639e4a5e8451a8ff04b89616 2024-11-21T21:24:58,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/06ca25c7e4644bcf91c7757cd8e7235b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/06ca25c7e4644bcf91c7757cd8e7235b 2024-11-21T21:24:58,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2356493c88b24db999ebd8924a4bbbaf to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2356493c88b24db999ebd8924a4bbbaf 2024-11-21T21:24:58,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2f2a6c5e670e46ccb98ad3c9e934e753 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2f2a6c5e670e46ccb98ad3c9e934e753 2024-11-21T21:24:58,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/8434250f9d0f4735bdc5206eb6a315d8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/8434250f9d0f4735bdc5206eb6a315d8 2024-11-21T21:24:58,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/1e83c0c0a0fb4c6fb2efe9780a518e8c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/1e83c0c0a0fb4c6fb2efe9780a518e8c 2024-11-21T21:24:58,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f2a18c1801054380aa68e59b60a45193 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/f2a18c1801054380aa68e59b60a45193 2024-11-21T21:24:58,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/83bf59be1bc14a37bad6f4d30ed1942b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/83bf59be1bc14a37bad6f4d30ed1942b 2024-11-21T21:24:58,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb75cacdf04e4af9b58b3880df513c39 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/cb75cacdf04e4af9b58b3880df513c39 2024-11-21T21:24:58,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/3db9359dff1c48ddac37f2172b01de54 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/3db9359dff1c48ddac37f2172b01de54 2024-11-21T21:24:58,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/efdf4122597148d6a4f1cc9a9f7b144e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/efdf4122597148d6a4f1cc9a9f7b144e 2024-11-21T21:24:58,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ee0c385d5dec4b9b867eb59843f6e803 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ee0c385d5dec4b9b867eb59843f6e803 2024-11-21T21:24:58,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/077e7f17e5ba44ec8afa12e0461e1f72 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/077e7f17e5ba44ec8afa12e0461e1f72 2024-11-21T21:24:58,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ba9b26275e13456e83b50af3a8e3eb28 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/ba9b26275e13456e83b50af3a8e3eb28 2024-11-21T21:24:58,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/79a16053880e404e8ae2a7019271ab4e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/79a16053880e404e8ae2a7019271ab4e 2024-11-21T21:24:58,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/77136c1e55e24cc691609dc5d6e61036 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/77136c1e55e24cc691609dc5d6e61036 2024-11-21T21:24:58,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fb3a4f94d4ef47eeb1770fd2439f26d9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/fb3a4f94d4ef47eeb1770fd2439f26d9 2024-11-21T21:24:58,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6628524c103c435c8db0fbe886d1fb07 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/6628524c103c435c8db0fbe886d1fb07 2024-11-21T21:24:58,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2393216d42be4626804e634bfe6be1c6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/2393216d42be4626804e634bfe6be1c6 2024-11-21T21:24:58,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/04f1b8dd73fb4eb1bb24eff3fa84219b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b0bbed8b536f451e9fc5ced28aa6917b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0d77f11b57f24653a717093dd71b44ce, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/4c0d15c6b09c4fe4a0226bf342e9a458, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/30185d0c344b4b9595b3c0d15d12152f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3c5147b7afd44e53a44c0666c768f04b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/e96ca9bd774f4817967e86796b99388f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/49267ce4b4d74876baab7259b5e0ce29, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8db0fbca48b3462c910b48852097721f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f2e6f498f2824b0c890aa1fffad0dd44, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/87b035e93d554e62a5dc5a37b99c4494, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/d3866d3d0ac44ea788247a98ddf8abcf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/ab1af2ef2d5343faa4a662aff1fef6bc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/06e7f3cb4e954a6baa6f067e9f23256f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/558bc66a39144d13a8948307f21e1c17, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/97a159f8498149f5a6c9d21a8d93ab1c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/35dca5d5d1954c9b9fd2c76ce125a02c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9f94f9a21cf8402790226892d85558a1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/758adc1a08b84ca0bcb4f51435f4b6b2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/92ceb6089d0f45d19ebdb65ab3fb3c5c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/41196276a28a4145b8667912613be00c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0534f5307967415ab09ea44133b8626a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/acc8c6909bc8414e958387b0e26f174a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/627e55fc030845d7be8032530c17bb09, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/adb07bde536943fb8ea5146150baa7a1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/01e66afdccc94b358b11b3f3bdf852f4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3da4f328b84043099dc9266cdc9c6b38, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f59ec9e4970b4847b530e503c0637a9b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9b281355033d45b09219f812f83c4953] to archive 2024-11-21T21:24:58,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:24:58,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/04f1b8dd73fb4eb1bb24eff3fa84219b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/04f1b8dd73fb4eb1bb24eff3fa84219b 2024-11-21T21:24:58,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b0bbed8b536f451e9fc5ced28aa6917b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b0bbed8b536f451e9fc5ced28aa6917b 2024-11-21T21:24:58,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0d77f11b57f24653a717093dd71b44ce to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0d77f11b57f24653a717093dd71b44ce 2024-11-21T21:24:58,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/4c0d15c6b09c4fe4a0226bf342e9a458 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/4c0d15c6b09c4fe4a0226bf342e9a458 2024-11-21T21:24:58,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/30185d0c344b4b9595b3c0d15d12152f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/30185d0c344b4b9595b3c0d15d12152f 2024-11-21T21:24:58,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3c5147b7afd44e53a44c0666c768f04b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3c5147b7afd44e53a44c0666c768f04b 2024-11-21T21:24:58,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/e96ca9bd774f4817967e86796b99388f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/e96ca9bd774f4817967e86796b99388f 2024-11-21T21:24:58,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/49267ce4b4d74876baab7259b5e0ce29 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/49267ce4b4d74876baab7259b5e0ce29 2024-11-21T21:24:58,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8db0fbca48b3462c910b48852097721f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8db0fbca48b3462c910b48852097721f 2024-11-21T21:24:58,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f2e6f498f2824b0c890aa1fffad0dd44 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f2e6f498f2824b0c890aa1fffad0dd44 2024-11-21T21:24:58,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/87b035e93d554e62a5dc5a37b99c4494 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/87b035e93d554e62a5dc5a37b99c4494 2024-11-21T21:24:58,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/d3866d3d0ac44ea788247a98ddf8abcf to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/d3866d3d0ac44ea788247a98ddf8abcf 2024-11-21T21:24:58,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/ab1af2ef2d5343faa4a662aff1fef6bc to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/ab1af2ef2d5343faa4a662aff1fef6bc 2024-11-21T21:24:58,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/06e7f3cb4e954a6baa6f067e9f23256f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/06e7f3cb4e954a6baa6f067e9f23256f 2024-11-21T21:24:58,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/558bc66a39144d13a8948307f21e1c17 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/558bc66a39144d13a8948307f21e1c17 2024-11-21T21:24:58,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/97a159f8498149f5a6c9d21a8d93ab1c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/97a159f8498149f5a6c9d21a8d93ab1c 2024-11-21T21:24:58,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/35dca5d5d1954c9b9fd2c76ce125a02c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/35dca5d5d1954c9b9fd2c76ce125a02c 2024-11-21T21:24:58,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9f94f9a21cf8402790226892d85558a1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9f94f9a21cf8402790226892d85558a1 2024-11-21T21:24:58,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/758adc1a08b84ca0bcb4f51435f4b6b2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/758adc1a08b84ca0bcb4f51435f4b6b2 2024-11-21T21:24:58,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/92ceb6089d0f45d19ebdb65ab3fb3c5c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/92ceb6089d0f45d19ebdb65ab3fb3c5c 2024-11-21T21:24:58,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/41196276a28a4145b8667912613be00c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/41196276a28a4145b8667912613be00c 2024-11-21T21:24:58,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0534f5307967415ab09ea44133b8626a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/0534f5307967415ab09ea44133b8626a 2024-11-21T21:24:58,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/8d5fe5c648c2420e8fa8a0c3a8afcc0a 2024-11-21T21:24:58,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/acc8c6909bc8414e958387b0e26f174a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/acc8c6909bc8414e958387b0e26f174a 2024-11-21T21:24:58,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/627e55fc030845d7be8032530c17bb09 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/627e55fc030845d7be8032530c17bb09 2024-11-21T21:24:58,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/adb07bde536943fb8ea5146150baa7a1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/adb07bde536943fb8ea5146150baa7a1 2024-11-21T21:24:58,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/01e66afdccc94b358b11b3f3bdf852f4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/01e66afdccc94b358b11b3f3bdf852f4 2024-11-21T21:24:58,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3da4f328b84043099dc9266cdc9c6b38 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/3da4f328b84043099dc9266cdc9c6b38 2024-11-21T21:24:58,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f59ec9e4970b4847b530e503c0637a9b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/f59ec9e4970b4847b530e503c0637a9b 2024-11-21T21:24:58,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/b2d669a78ef84f1dbbcbbcfcb2e27c9e 2024-11-21T21:24:58,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9b281355033d45b09219f812f83c4953 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/9b281355033d45b09219f812f83c4953 2024-11-21T21:24:58,430 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=4 2024-11-21T21:24:58,431 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. 2024-11-21T21:24:58,431 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for cd5da8e0c287760dca4b3d875dd531a4: Waiting for close lock at 1732224296667Running coprocessor pre-close hooks at 1732224296668 (+1 ms)Disabling compacts and flushes for region at 1732224296668Disabling writes for close at 1732224296668Obtaining lock to block concurrent updates at 1732224296668Preparing flush snapshotting stores in cd5da8e0c287760dca4b3d875dd531a4 at 1732224296668Finished memstore snapshotting TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732224296668Flushing stores of TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4. at 1732224296672 (+4 ms)Flushing cd5da8e0c287760dca4b3d875dd531a4/A: creating writer at 1732224296672Flushing mob file cd5da8e0c287760dca4b3d875dd531a4/A: appending metadata at 1732224296676 (+4 ms)Flushing mob file cd5da8e0c287760dca4b3d875dd531a4/A: closing flushed file at 1732224296676Flushing cd5da8e0c287760dca4b3d875dd531a4/A: appending metadata at 1732224297090 (+414 ms)Flushing cd5da8e0c287760dca4b3d875dd531a4/A: closing flushed file at 1732224297090Flushing cd5da8e0c287760dca4b3d875dd531a4/B: creating writer at 1732224297500 (+410 ms)Flushing cd5da8e0c287760dca4b3d875dd531a4/B: appending metadata at 1732224297502 (+2 ms)Flushing cd5da8e0c287760dca4b3d875dd531a4/B: closing flushed file at 1732224297502Flushing cd5da8e0c287760dca4b3d875dd531a4/C: creating writer at 1732224297914 (+412 ms)Flushing cd5da8e0c287760dca4b3d875dd531a4/C: appending metadata at 1732224297916 (+2 ms)Flushing cd5da8e0c287760dca4b3d875dd531a4/C: closing flushed file at 1732224297916Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fd6704a: reopening flushed file at 1732224298324 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b2adc36: reopening flushed file at 1732224298329 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2142e855: reopening flushed file at 1732224298333 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for cd5da8e0c287760dca4b3d875dd531a4 in 1671ms, sequenceid=465, compaction requested=true at 1732224298339 (+6 ms)Writing region close event to WAL at 1732224298427 (+88 ms)Running coprocessor post-close hooks at 1732224298430 (+3 ms)Closed at 1732224298431 (+1 ms) 2024-11-21T21:24:58,433 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,433 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cd5da8e0c287760dca4b3d875dd531a4, regionState=CLOSED 2024-11-21T21:24:58,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:24:58,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T21:24:58,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure cd5da8e0c287760dca4b3d875dd531a4, server=849dee24a242,34123,1732224229084 in 1.9210 sec 2024-11-21T21:24:58,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-21T21:24:58,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cd5da8e0c287760dca4b3d875dd531a4, UNASSIGN in 1.9260 sec 2024-11-21T21:24:58,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T21:24:58,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9290 sec 2024-11-21T21:24:58,440 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224298440"}]},"ts":"1732224298440"} 2024-11-21T21:24:58,441 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:24:58,441 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:24:58,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9390 sec 2024-11-21T21:24:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:24:58,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:58,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:24:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:58,656 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T21:24:58,658 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:58,664 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,666 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/recovered.edits] 2024-11-21T21:24:58,669 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/6b7c8463c90c4650a3332d484f2711c4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/6b7c8463c90c4650a3332d484f2711c4 2024-11-21T21:24:58,670 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/861159a9f3b54d849a38a46393a7c62b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/861159a9f3b54d849a38a46393a7c62b 2024-11-21T21:24:58,671 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d8b1beeeaed84130bd6f982a6575649a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/A/d8b1beeeaed84130bd6f982a6575649a 2024-11-21T21:24:58,673 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/010d4ee569ed4827b7ad187d0d594806 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/010d4ee569ed4827b7ad187d0d594806 2024-11-21T21:24:58,674 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/7a4a7b2e17ff4eb1911659c1171fc283 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/7a4a7b2e17ff4eb1911659c1171fc283 2024-11-21T21:24:58,675 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/dd9a95ba7ba1454e807ac57d707fde52 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/B/dd9a95ba7ba1454e807ac57d707fde52 2024-11-21T21:24:58,677 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/25c6704545424224b2c0fd9082a7aea1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/25c6704545424224b2c0fd9082a7aea1 2024-11-21T21:24:58,678 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/7f9d86ac1fcb43fda8b0f1bb525159fd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/7f9d86ac1fcb43fda8b0f1bb525159fd 2024-11-21T21:24:58,679 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/dbff31937bc542feaf35bf7178a46ecc to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/C/dbff31937bc542feaf35bf7178a46ecc 2024-11-21T21:24:58,681 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/recovered.edits/468.seqid to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4/recovered.edits/468.seqid 2024-11-21T21:24:58,681 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,681 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:24:58,681 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:24:58,682 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T21:24:58,685 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210f4a18f4a7fe4f50b9a39af45fcaa3d4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210f4a18f4a7fe4f50b9a39af45fcaa3d4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,686 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121179ea192120f477982eab4a0b6b4722a_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121179ea192120f477982eab4a0b6b4722a_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,687 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211c4d9036f91c4e5abd370e61878d9f7f_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211c4d9036f91c4e5abd370e61878d9f7f_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,687 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121269e37c3c7624dde87b5d5b165a64317_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121269e37c3c7624dde87b5d5b165a64317_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,688 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ac91afae8394d69ace03d6e2fc74ded_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ac91afae8394d69ace03d6e2fc74ded_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,689 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121319ddf843aa64652a075f0afcdf05f3b_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121319ddf843aa64652a075f0afcdf05f3b_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,690 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121364cfe5ff5ca4ff087854e96cc16b00b_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121364cfe5ff5ca4ff087854e96cc16b00b_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,691 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213983e5523e5c4f23a75ec6da5a713fbe_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213983e5523e5c4f23a75ec6da5a713fbe_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,692 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112149c43d9b6e6c4c9cad09a3ce8fac9d21_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112149c43d9b6e6c4c9cad09a3ce8fac9d21_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,693 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214d3f4658cd1c4331969d081b87fa02c4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214d3f4658cd1c4331969d081b87fa02c4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,694 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214e8365a1bfe34a7f841dee19eca83fc7_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214e8365a1bfe34a7f841dee19eca83fc7_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,695 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f4fd9a30ccd4e6886c04d7263c68f8f_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f4fd9a30ccd4e6886c04d7263c68f8f_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,695 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112159110da5491740bf869002df365f9ea7_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112159110da5491740bf869002df365f9ea7_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,696 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215f171481c4614367980c96a7f20476c4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215f171481c4614367980c96a7f20476c4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,697 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112163da21cd3ffd439d8f9653a67da32d34_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112163da21cd3ffd439d8f9653a67da32d34_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,698 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218c7abceb4a93472b943a0628a7ea415a_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218c7abceb4a93472b943a0628a7ea415a_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,699 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e66e9bd618f4cf9a4d65be8837b337f_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e66e9bd618f4cf9a4d65be8837b337f_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,700 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a93923685fa24f90913287fe99a61d37_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a93923685fa24f90913287fe99a61d37_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,701 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bdfe61723c2246fd8e9517f2f4ca4023_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bdfe61723c2246fd8e9517f2f4ca4023_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,702 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c3630a260dc34d2b97e6b9b125971710_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c3630a260dc34d2b97e6b9b125971710_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,703 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c929f60d9f74421abeac160ae4d9501d_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c929f60d9f74421abeac160ae4d9501d_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,704 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cacf31fd7ee34eecb8d91f4cd7dbe7bc_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cacf31fd7ee34eecb8d91f4cd7dbe7bc_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,705 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e17ff83db12c4044aa32aaa3205bef83_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e17ff83db12c4044aa32aaa3205bef83_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,706 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4544e2a9bad46f08a5eacee35ee98f4_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4544e2a9bad46f08a5eacee35ee98f4_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,707 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f7dada38c2284df7a9e155e5f933e8cb_cd5da8e0c287760dca4b3d875dd531a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f7dada38c2284df7a9e155e5f933e8cb_cd5da8e0c287760dca4b3d875dd531a4 2024-11-21T21:24:58,707 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:24:58,709 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:58,711 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:24:58,713 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:24:58,714 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:58,714 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:24:58,714 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732224298714"}]},"ts":"9223372036854775807"} 2024-11-21T21:24:58,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:24:58,716 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cd5da8e0c287760dca4b3d875dd531a4, NAME => 'TestAcidGuarantees,,1732224265011.cd5da8e0c287760dca4b3d875dd531a4.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:24:58,716 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:24:58,716 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732224298716"}]},"ts":"9223372036854775807"} 2024-11-21T21:24:58,718 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:24:58,718 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:58,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-21T21:24:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T21:24:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:24:58,770 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:24:58,779 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1390536951_22 at /127.0.0.1:39352 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 448) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12295 (was 12142) - AvailableMemoryMB LEAK? - 2024-11-21T21:24:58,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=12294 2024-11-21T21:24:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:24:58,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:24:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:24:58,790 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:24:58,790 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:58,790 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-21T21:24:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T21:24:58,791 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:24:58,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742087_1263 (size=959) 2024-11-21T21:24:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T21:24:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T21:24:59,198 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:24:59,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742088_1264 (size=53) 2024-11-21T21:24:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T21:24:59,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:24:59,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d6ccac84b1b168c8055162a63f397098, disabling compactions & flushes 2024-11-21T21:24:59,604 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:24:59,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:24:59,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. after waiting 0 ms 2024-11-21T21:24:59,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:24:59,604 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:24:59,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d6ccac84b1b168c8055162a63f397098: Waiting for close lock at 1732224299604Disabling compacts and flushes for region at 1732224299604Disabling writes for close at 1732224299604Writing region close event to WAL at 1732224299604Closed at 1732224299604 2024-11-21T21:24:59,605 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:24:59,605 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732224299605"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732224299605"}]},"ts":"1732224299605"} 2024-11-21T21:24:59,607 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:24:59,608 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:24:59,608 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224299608"}]},"ts":"1732224299608"} 2024-11-21T21:24:59,609 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:24:59,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, ASSIGN}] 2024-11-21T21:24:59,611 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, ASSIGN 2024-11-21T21:24:59,611 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, ASSIGN; state=OFFLINE, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=false 2024-11-21T21:24:59,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d6ccac84b1b168c8055162a63f397098, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:59,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, ASSIGN because future has completed 2024-11-21T21:24:59,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:24:59,925 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:24:59,926 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:24:59,926 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,927 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:24:59,927 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,927 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,930 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T21:24:59,932 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6ccac84b1b168c8055162a63f397098 columnFamilyName A 2024-11-21T21:24:59,932 DEBUG [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:59,933 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] regionserver.HStore(327): Store=d6ccac84b1b168c8055162a63f397098/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:59,933 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,934 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6ccac84b1b168c8055162a63f397098 columnFamilyName B 2024-11-21T21:24:59,934 DEBUG [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:59,935 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] regionserver.HStore(327): Store=d6ccac84b1b168c8055162a63f397098/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:59,935 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,936 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6ccac84b1b168c8055162a63f397098 columnFamilyName C 2024-11-21T21:24:59,936 DEBUG [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:24:59,937 INFO [StoreOpener-d6ccac84b1b168c8055162a63f397098-1 {}] regionserver.HStore(327): Store=d6ccac84b1b168c8055162a63f397098/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:24:59,937 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,938 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,938 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,939 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,939 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,940 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:24:59,941 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,943 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:24:59,943 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened d6ccac84b1b168c8055162a63f397098; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58761334, jitterRate=-0.12438789010047913}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:24:59,944 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6ccac84b1b168c8055162a63f397098 2024-11-21T21:24:59,944 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for d6ccac84b1b168c8055162a63f397098: Running coprocessor pre-open hook at 1732224299927Writing region info on filesystem at 1732224299927Initializing all the Stores at 1732224299929 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224299929Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224299930 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224299930Cleaning up temporary data from old regions at 1732224299939 (+9 ms)Running coprocessor post-open hooks at 1732224299944 (+5 ms)Region opened successfully at 1732224299944 2024-11-21T21:24:59,945 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., pid=61, masterSystemTime=1732224299918 2024-11-21T21:24:59,947 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:24:59,947 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:24:59,947 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d6ccac84b1b168c8055162a63f397098, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:24:59,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:24:59,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T21:24:59,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 in 184 msec 2024-11-21T21:24:59,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-21T21:24:59,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, ASSIGN in 342 msec 2024-11-21T21:24:59,954 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:24:59,955 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224299954"}]},"ts":"1732224299954"} 2024-11-21T21:24:59,956 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:24:59,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:24:59,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-21T21:25:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T21:25:00,940 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:00,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,943 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a47d9cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,945 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9eb835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:00,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:00,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:00,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:00,949 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:25:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,35047,1732224227985 2024-11-21T21:25:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@370add08 2024-11-21T21:25:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:25:00,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:25:00,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0f7e3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,954 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edf7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,956 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8219ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,958 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2884d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,960 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b999b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,961 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cce762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d92d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,965 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dadc215, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,968 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dd0eb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,970 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@434322e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,972 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d56a3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,973 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781938de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,976 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e53e79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,977 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451e420c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,980 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6ad412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,981 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465c3d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@466e136a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254240f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:00,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:00,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55064a92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:00,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2963db91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:00,991 DEBUG [Thread-1220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:00,992 DEBUG [Thread-1237 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:00,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:00,994 DEBUG [Thread-1226 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:00,994 DEBUG [Thread-1222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:00,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:00,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:00,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:00,998 DEBUG [Thread-1235 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:00,999 DEBUG [Thread-1233 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:01,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:01,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:01,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:01,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-21T21:25:01,005 DEBUG [Thread-1218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T21:25:01,005 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:01,006 DEBUG [Thread-1224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:01,006 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:01,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:01,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:01,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:01,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:25:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:01,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:01,028 DEBUG [Thread-1231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:01,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224361030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224361031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224361033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224361033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224361029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,044 DEBUG [Thread-1229 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:01,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:01,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3bc21805f4034d35a13b8d8fcb29e90f is 50, key is test_row_0/A:col10/1732224301008/Put/seqid=0 2024-11-21T21:25:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742089_1265 (size=12001) 2024-11-21T21:25:01,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3bc21805f4034d35a13b8d8fcb29e90f 2024-11-21T21:25:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T21:25:01,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/40a1ca7988a64b9285f8796ae847d74d is 50, key is test_row_0/B:col10/1732224301008/Put/seqid=0 2024-11-21T21:25:01,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224361140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224361141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224361140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224361141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224361149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T21:25:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:01,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:01,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742090_1266 (size=12001) 2024-11-21T21:25:01,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/40a1ca7988a64b9285f8796ae847d74d 2024-11-21T21:25:01,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ec5a3500777b4f488cd9560f6c6b3120 is 50, key is test_row_0/C:col10/1732224301008/Put/seqid=0 2024-11-21T21:25:01,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742091_1267 (size=12001) 2024-11-21T21:25:01,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ec5a3500777b4f488cd9560f6c6b3120 2024-11-21T21:25:01,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3bc21805f4034d35a13b8d8fcb29e90f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3bc21805f4034d35a13b8d8fcb29e90f 2024-11-21T21:25:01,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3bc21805f4034d35a13b8d8fcb29e90f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T21:25:01,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/40a1ca7988a64b9285f8796ae847d74d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/40a1ca7988a64b9285f8796ae847d74d 2024-11-21T21:25:01,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/40a1ca7988a64b9285f8796ae847d74d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T21:25:01,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ec5a3500777b4f488cd9560f6c6b3120 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ec5a3500777b4f488cd9560f6c6b3120 2024-11-21T21:25:01,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ec5a3500777b4f488cd9560f6c6b3120, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T21:25:01,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d6ccac84b1b168c8055162a63f397098 in 278ms, sequenceid=13, compaction requested=false 2024-11-21T21:25:01,287 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T21:25:01,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:01,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T21:25:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:01,316 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:25:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T21:25:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/fdd8d6474f954db98954356f96713a93 is 50, key is test_row_0/A:col10/1732224301029/Put/seqid=0 2024-11-21T21:25:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:01,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:01,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224361365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224361363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224361365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224361367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224361367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742092_1268 (size=12001) 2024-11-21T21:25:01,389 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/fdd8d6474f954db98954356f96713a93 2024-11-21T21:25:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3b5e0460e0214b07a6d4561c9106237b is 50, key is test_row_0/B:col10/1732224301029/Put/seqid=0 2024-11-21T21:25:01,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742093_1269 (size=12001) 2024-11-21T21:25:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224361479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224361479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224361479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224361479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224361479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T21:25:01,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224361690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224361690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224361690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224361690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224361690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:01,850 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3b5e0460e0214b07a6d4561c9106237b 2024-11-21T21:25:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/fd5cb062ac794e418d5554324663ac16 is 50, key is test_row_0/C:col10/1732224301029/Put/seqid=0 2024-11-21T21:25:01,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742094_1270 (size=12001) 2024-11-21T21:25:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224362000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224362000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224362000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224362000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224362000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T21:25:02,263 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/fd5cb062ac794e418d5554324663ac16 2024-11-21T21:25:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/fdd8d6474f954db98954356f96713a93 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/fdd8d6474f954db98954356f96713a93 2024-11-21T21:25:02,274 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/fdd8d6474f954db98954356f96713a93, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T21:25:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3b5e0460e0214b07a6d4561c9106237b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3b5e0460e0214b07a6d4561c9106237b 2024-11-21T21:25:02,281 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3b5e0460e0214b07a6d4561c9106237b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T21:25:02,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/fd5cb062ac794e418d5554324663ac16 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/fd5cb062ac794e418d5554324663ac16 2024-11-21T21:25:02,287 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/fd5cb062ac794e418d5554324663ac16, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T21:25:02,289 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d6ccac84b1b168c8055162a63f397098 in 973ms, sequenceid=37, compaction requested=false 2024-11-21T21:25:02,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:02,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:02,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-21T21:25:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-21T21:25:02,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T21:25:02,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2850 sec 2024-11-21T21:25:02,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.2920 sec 2024-11-21T21:25:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:02,512 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:02,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:25:02,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/ae9699ffc53f4734b152c6d267880b2d is 50, key is test_row_0/A:col10/1732224301364/Put/seqid=0 2024-11-21T21:25:02,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224362539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224362539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224362541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224362543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224362545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742095_1271 (size=12001) 2024-11-21T21:25:02,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/ae9699ffc53f4734b152c6d267880b2d 2024-11-21T21:25:02,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/53dd08fc2de44bd6bb1988d1631fbdee is 50, key is test_row_0/B:col10/1732224301364/Put/seqid=0 2024-11-21T21:25:02,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742096_1272 (size=12001) 2024-11-21T21:25:02,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224362650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224362650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224362650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224362659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224362659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224362860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224362860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224362860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224362869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224362869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:02,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/53dd08fc2de44bd6bb1988d1631fbdee 2024-11-21T21:25:03,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/389d7cd39d4249339538d5325e9adb3d is 50, key is test_row_0/C:col10/1732224301364/Put/seqid=0 2024-11-21T21:25:03,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742097_1273 (size=12001) 2024-11-21T21:25:03,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/389d7cd39d4249339538d5325e9adb3d 2024-11-21T21:25:03,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/ae9699ffc53f4734b152c6d267880b2d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/ae9699ffc53f4734b152c6d267880b2d 2024-11-21T21:25:03,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/ae9699ffc53f4734b152c6d267880b2d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T21:25:03,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/53dd08fc2de44bd6bb1988d1631fbdee as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/53dd08fc2de44bd6bb1988d1631fbdee 2024-11-21T21:25:03,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/53dd08fc2de44bd6bb1988d1631fbdee, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T21:25:03,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/389d7cd39d4249339538d5325e9adb3d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/389d7cd39d4249339538d5325e9adb3d 2024-11-21T21:25:03,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/389d7cd39d4249339538d5325e9adb3d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T21:25:03,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d6ccac84b1b168c8055162a63f397098 in 550ms, sequenceid=53, compaction requested=true 2024-11-21T21:25:03,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:03,063 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:03,063 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:03,064 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,064 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,064 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/40a1ca7988a64b9285f8796ae847d74d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3b5e0460e0214b07a6d4561c9106237b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/53dd08fc2de44bd6bb1988d1631fbdee] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=35.2 K 2024-11-21T21:25:03,064 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3bc21805f4034d35a13b8d8fcb29e90f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/fdd8d6474f954db98954356f96713a93, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/ae9699ffc53f4734b152c6d267880b2d] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=35.2 K 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc21805f4034d35a13b8d8fcb29e90f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732224301007 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a1ca7988a64b9285f8796ae847d74d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732224301007 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdd8d6474f954db98954356f96713a93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732224301029 2024-11-21T21:25:03,064 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5e0460e0214b07a6d4561c9106237b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732224301029 2024-11-21T21:25:03,065 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 53dd08fc2de44bd6bb1988d1631fbdee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732224301362 2024-11-21T21:25:03,065 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae9699ffc53f4734b152c6d267880b2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732224301362 2024-11-21T21:25:03,078 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:03,078 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/92d81f799f104638b04aa928bfbaf17c is 50, key is test_row_0/B:col10/1732224301364/Put/seqid=0 2024-11-21T21:25:03,082 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:03,082 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/6bd9ebdaccff41a7be217e7dcea3b79f is 50, key is test_row_0/A:col10/1732224301364/Put/seqid=0 2024-11-21T21:25:03,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742099_1275 (size=12104) 2024-11-21T21:25:03,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742098_1274 (size=12104) 2024-11-21T21:25:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T21:25:03,160 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:03,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-21T21:25:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:25:03,164 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:03,165 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:03,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:03,174 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:03,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:25:03,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0d3699e122914cfca38a1d3799cdd49f is 50, key is test_row_0/A:col10/1732224303172/Put/seqid=0 2024-11-21T21:25:03,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224363186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224363190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224363190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224363191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224363191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742100_1276 (size=16681) 2024-11-21T21:25:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:25:03,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224363299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224363300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224363300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224363300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224363300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:25:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:25:03,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224363510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224363510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224363510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224363510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224363510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,523 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/6bd9ebdaccff41a7be217e7dcea3b79f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/6bd9ebdaccff41a7be217e7dcea3b79f 2024-11-21T21:25:03,527 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/92d81f799f104638b04aa928bfbaf17c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/92d81f799f104638b04aa928bfbaf17c 2024-11-21T21:25:03,528 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into 6bd9ebdaccff41a7be217e7dcea3b79f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:03,528 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:03,528 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=13, startTime=1732224303063; duration=0sec 2024-11-21T21:25:03,528 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:03,528 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:03,528 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:03,533 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:03,533 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:03,533 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,533 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ec5a3500777b4f488cd9560f6c6b3120, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/fd5cb062ac794e418d5554324663ac16, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/389d7cd39d4249339538d5325e9adb3d] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=35.2 K 2024-11-21T21:25:03,533 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec5a3500777b4f488cd9560f6c6b3120, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732224301007 2024-11-21T21:25:03,534 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd5cb062ac794e418d5554324663ac16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732224301029 2024-11-21T21:25:03,534 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389d7cd39d4249339538d5325e9adb3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732224301362 2024-11-21T21:25:03,536 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into 92d81f799f104638b04aa928bfbaf17c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:03,536 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:03,536 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=13, startTime=1732224303063; duration=0sec 2024-11-21T21:25:03,537 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:03,537 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:03,544 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:03,544 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/8a8e17f75d5c4c02a973c897992b0105 is 50, key is test_row_0/C:col10/1732224301364/Put/seqid=0 2024-11-21T21:25:03,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742101_1277 (size=12104) 2024-11-21T21:25:03,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0d3699e122914cfca38a1d3799cdd49f 2024-11-21T21:25:03,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/0e8e6b6caeae436eb7f0f62189f1e8f2 is 50, key is test_row_0/B:col10/1732224303172/Put/seqid=0 2024-11-21T21:25:03,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742102_1278 (size=12001) 2024-11-21T21:25:03,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/0e8e6b6caeae436eb7f0f62189f1e8f2 2024-11-21T21:25:03,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:25:03,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/5be85ae02dea466b8c965f033569a69b is 50, key is test_row_0/C:col10/1732224303172/Put/seqid=0 2024-11-21T21:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742103_1279 (size=12001) 2024-11-21T21:25:03,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:25:03,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:03,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:25:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224363819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224363820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224363820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224363820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224363820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:03,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:25:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:03,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:03,952 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/8a8e17f75d5c4c02a973c897992b0105 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/8a8e17f75d5c4c02a973c897992b0105 2024-11-21T21:25:03,956 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into 8a8e17f75d5c4c02a973c897992b0105(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:03,956 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:03,956 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=13, startTime=1732224303063; duration=0sec 2024-11-21T21:25:03,956 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:03,956 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:03,963 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:25:04,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/5be85ae02dea466b8c965f033569a69b 2024-11-21T21:25:04,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0d3699e122914cfca38a1d3799cdd49f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d3699e122914cfca38a1d3799cdd49f 2024-11-21T21:25:04,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d3699e122914cfca38a1d3799cdd49f, entries=250, sequenceid=75, filesize=16.3 K 2024-11-21T21:25:04,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/0e8e6b6caeae436eb7f0f62189f1e8f2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/0e8e6b6caeae436eb7f0f62189f1e8f2 2024-11-21T21:25:04,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/0e8e6b6caeae436eb7f0f62189f1e8f2, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T21:25:04,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/5be85ae02dea466b8c965f033569a69b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/5be85ae02dea466b8c965f033569a69b 2024-11-21T21:25:04,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/5be85ae02dea466b8c965f033569a69b, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T21:25:04,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d6ccac84b1b168c8055162a63f397098 in 872ms, sequenceid=75, compaction requested=false 2024-11-21T21:25:04,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:04,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:25:04,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:04,083 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:25:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/c0fef7e53c1f4b5fa206bb217476b002 is 50, key is test_row_0/A:col10/1732224303189/Put/seqid=0 2024-11-21T21:25:04,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742104_1280 (size=12001) 2024-11-21T21:25:04,113 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/c0fef7e53c1f4b5fa206bb217476b002 2024-11-21T21:25:04,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/72d3f9e7d1ee4280949b85de9447503e is 50, key is test_row_0/B:col10/1732224303189/Put/seqid=0 2024-11-21T21:25:04,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742105_1281 (size=12001) 2024-11-21T21:25:04,131 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/72d3f9e7d1ee4280949b85de9447503e 2024-11-21T21:25:04,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ee6809bf50bf4ca3a6094e3fdaee41a7 is 50, key is test_row_0/C:col10/1732224303189/Put/seqid=0 2024-11-21T21:25:04,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742106_1282 (size=12001) 2024-11-21T21:25:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:25:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:04,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:04,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224364347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224364347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224364348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224364349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224364350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224364459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224364460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224364460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224364460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224364460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,541 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ee6809bf50bf4ca3a6094e3fdaee41a7 2024-11-21T21:25:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/c0fef7e53c1f4b5fa206bb217476b002 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/c0fef7e53c1f4b5fa206bb217476b002 2024-11-21T21:25:04,550 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/c0fef7e53c1f4b5fa206bb217476b002, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T21:25:04,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/72d3f9e7d1ee4280949b85de9447503e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/72d3f9e7d1ee4280949b85de9447503e 2024-11-21T21:25:04,555 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/72d3f9e7d1ee4280949b85de9447503e, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T21:25:04,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ee6809bf50bf4ca3a6094e3fdaee41a7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ee6809bf50bf4ca3a6094e3fdaee41a7 2024-11-21T21:25:04,559 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ee6809bf50bf4ca3a6094e3fdaee41a7, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T21:25:04,560 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d6ccac84b1b168c8055162a63f397098 in 477ms, sequenceid=92, compaction requested=true 2024-11-21T21:25:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-21T21:25:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-21T21:25:04,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-21T21:25:04,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3970 sec 2024-11-21T21:25:04,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4030 sec 2024-11-21T21:25:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:04,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:04,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:25:04,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/796d441b4f434ff09956d01561dd597d is 50, key is test_row_0/A:col10/1732224304672/Put/seqid=0 2024-11-21T21:25:04,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742107_1283 (size=12001) 2024-11-21T21:25:04,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224364680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224364680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224364682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224364682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224364686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224364789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224364789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224364789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224364799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224364799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224364999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224364999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224364999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224365009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224365009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/796d441b4f434ff09956d01561dd597d 2024-11-21T21:25:05,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/06541a3aa91845cfb0d156974677a18b is 50, key is test_row_0/B:col10/1732224304672/Put/seqid=0 2024-11-21T21:25:05,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742108_1284 (size=12001) 2024-11-21T21:25:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:25:05,310 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:05,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-21T21:25:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:25:05,312 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:05,312 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:05,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224365310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224365310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224365310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224365319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224365319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:25:05,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T21:25:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:05,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:05,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:05,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:05,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/06541a3aa91845cfb0d156974677a18b 2024-11-21T21:25:05,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/7c581dc42a654542a73562a15a4e20a7 is 50, key is test_row_0/C:col10/1732224304672/Put/seqid=0 2024-11-21T21:25:05,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742109_1285 (size=12001) 2024-11-21T21:25:05,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/7c581dc42a654542a73562a15a4e20a7 2024-11-21T21:25:05,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/796d441b4f434ff09956d01561dd597d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/796d441b4f434ff09956d01561dd597d 2024-11-21T21:25:05,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/796d441b4f434ff09956d01561dd597d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T21:25:05,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/06541a3aa91845cfb0d156974677a18b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/06541a3aa91845cfb0d156974677a18b 2024-11-21T21:25:05,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/06541a3aa91845cfb0d156974677a18b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T21:25:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/7c581dc42a654542a73562a15a4e20a7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7c581dc42a654542a73562a15a4e20a7 2024-11-21T21:25:05,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7c581dc42a654542a73562a15a4e20a7, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T21:25:05,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d6ccac84b1b168c8055162a63f397098 in 855ms, sequenceid=117, compaction requested=true 2024-11-21T21:25:05,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:05,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:05,528 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:05,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:05,529 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:05,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:05,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:05,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:05,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:05,530 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:05,530 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:05,530 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:05,530 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/92d81f799f104638b04aa928bfbaf17c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/0e8e6b6caeae436eb7f0f62189f1e8f2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/72d3f9e7d1ee4280949b85de9447503e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/06541a3aa91845cfb0d156974677a18b] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=47.0 K 2024-11-21T21:25:05,530 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:05,530 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:05,530 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:05,530 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/6bd9ebdaccff41a7be217e7dcea3b79f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d3699e122914cfca38a1d3799cdd49f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/c0fef7e53c1f4b5fa206bb217476b002, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/796d441b4f434ff09956d01561dd597d] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=51.5 K 2024-11-21T21:25:05,530 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d81f799f104638b04aa928bfbaf17c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732224301362 2024-11-21T21:25:05,531 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd9ebdaccff41a7be217e7dcea3b79f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732224301362 2024-11-21T21:25:05,531 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d3699e122914cfca38a1d3799cdd49f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732224302540 2024-11-21T21:25:05,531 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e8e6b6caeae436eb7f0f62189f1e8f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732224302542 2024-11-21T21:25:05,531 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0fef7e53c1f4b5fa206bb217476b002, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732224303180 2024-11-21T21:25:05,532 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 796d441b4f434ff09956d01561dd597d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732224304344 2024-11-21T21:25:05,532 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d3f9e7d1ee4280949b85de9447503e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732224303180 2024-11-21T21:25:05,532 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 06541a3aa91845cfb0d156974677a18b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732224304344 2024-11-21T21:25:05,541 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:05,542 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/fb25b3aa80f44de29f9896c9b6ba9e46 is 50, key is test_row_0/B:col10/1732224304672/Put/seqid=0 2024-11-21T21:25:05,542 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:05,542 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/541e79c11aa240b48112f26379d76812 is 50, key is test_row_0/A:col10/1732224304672/Put/seqid=0 2024-11-21T21:25:05,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742110_1286 (size=12241) 2024-11-21T21:25:05,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742111_1287 (size=12241) 2024-11-21T21:25:05,578 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/541e79c11aa240b48112f26379d76812 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/541e79c11aa240b48112f26379d76812 2024-11-21T21:25:05,583 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into 541e79c11aa240b48112f26379d76812(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:05,583 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:05,583 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=12, startTime=1732224305528; duration=0sec 2024-11-21T21:25:05,583 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:05,583 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:05,584 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:05,585 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:05,585 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:05,585 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:05,585 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/8a8e17f75d5c4c02a973c897992b0105, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/5be85ae02dea466b8c965f033569a69b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ee6809bf50bf4ca3a6094e3fdaee41a7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7c581dc42a654542a73562a15a4e20a7] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=47.0 K 2024-11-21T21:25:05,585 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8e17f75d5c4c02a973c897992b0105, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732224301362 2024-11-21T21:25:05,586 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be85ae02dea466b8c965f033569a69b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732224302542 2024-11-21T21:25:05,586 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee6809bf50bf4ca3a6094e3fdaee41a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732224303180 2024-11-21T21:25:05,586 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c581dc42a654542a73562a15a4e20a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732224304344 2024-11-21T21:25:05,595 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:05,595 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/40ac7b43f67a454cb6032b0f682d711a is 50, key is test_row_0/C:col10/1732224304672/Put/seqid=0 2024-11-21T21:25:05,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742112_1288 (size=12241) 2024-11-21T21:25:05,608 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/40ac7b43f67a454cb6032b0f682d711a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/40ac7b43f67a454cb6032b0f682d711a 2024-11-21T21:25:05,614 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into 40ac7b43f67a454cb6032b0f682d711a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:05,614 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:05,614 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=12, startTime=1732224305529; duration=0sec 2024-11-21T21:25:05,615 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:05,615 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:05,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T21:25:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:05,616 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:25:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/bf4a883379d947b08099faf0d12d627b is 50, key is test_row_0/A:col10/1732224304680/Put/seqid=0 2024-11-21T21:25:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:25:05,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742113_1289 (size=12001) 2024-11-21T21:25:05,635 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/bf4a883379d947b08099faf0d12d627b 2024-11-21T21:25:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/1fa1aee16d6c429ca09171ca8aeeb425 is 50, key is test_row_0/B:col10/1732224304680/Put/seqid=0 2024-11-21T21:25:05,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742114_1290 (size=12001) 2024-11-21T21:25:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:05,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:05,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224365841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224365843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224365843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224365845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224365846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:25:05,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224365950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224365950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224365950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224365959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224365959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:05,972 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/fb25b3aa80f44de29f9896c9b6ba9e46 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/fb25b3aa80f44de29f9896c9b6ba9e46 2024-11-21T21:25:05,976 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into fb25b3aa80f44de29f9896c9b6ba9e46(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:05,976 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:05,977 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=12, startTime=1732224305528; duration=0sec 2024-11-21T21:25:05,977 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:05,977 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:06,054 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/1fa1aee16d6c429ca09171ca8aeeb425 2024-11-21T21:25:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/49b3e9b9a0c348ff9b959ccea9e663fd is 50, key is test_row_0/C:col10/1732224304680/Put/seqid=0 2024-11-21T21:25:06,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742115_1291 (size=12001) 2024-11-21T21:25:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224366159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224366159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224366159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224366170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224366170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:25:06,465 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/49b3e9b9a0c348ff9b959ccea9e663fd 2024-11-21T21:25:06,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/bf4a883379d947b08099faf0d12d627b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bf4a883379d947b08099faf0d12d627b 2024-11-21T21:25:06,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224366469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224366469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224366469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,474 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bf4a883379d947b08099faf0d12d627b, entries=150, sequenceid=130, filesize=11.7 K 2024-11-21T21:25:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/1fa1aee16d6c429ca09171ca8aeeb425 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1fa1aee16d6c429ca09171ca8aeeb425 2024-11-21T21:25:06,479 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1fa1aee16d6c429ca09171ca8aeeb425, entries=150, sequenceid=130, filesize=11.7 K 2024-11-21T21:25:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/49b3e9b9a0c348ff9b959ccea9e663fd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/49b3e9b9a0c348ff9b959ccea9e663fd 2024-11-21T21:25:06,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224366480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224366480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,483 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/49b3e9b9a0c348ff9b959ccea9e663fd, entries=150, sequenceid=130, filesize=11.7 K 2024-11-21T21:25:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,484 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d6ccac84b1b168c8055162a63f397098 in 868ms, sequenceid=130, compaction requested=false 2024-11-21T21:25:06,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:06,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:06,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-21T21:25:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-21T21:25:06,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T21:25:06,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1730 sec 2024-11-21T21:25:06,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1770 sec 2024-11-21T21:25:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:06,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:06,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T21:25:06,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/60b3fdfb0ff240c4a0292985eb47cd27 is 50, key is test_row_0/A:col10/1732224305842/Put/seqid=0 2024-11-21T21:25:06,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224366988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742116_1292 (size=16931) 2024-11-21T21:25:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224366990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224366990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224366990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224366991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224367100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224367100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224367100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224367309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224367309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224367310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/60b3fdfb0ff240c4a0292985eb47cd27 2024-11-21T21:25:07,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/8945b366dccc40208ca5a1189a10e5e3 is 50, key is test_row_0/B:col10/1732224305842/Put/seqid=0 2024-11-21T21:25:07,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742117_1293 (size=12151) 2024-11-21T21:25:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:25:07,460 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:07,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T21:25:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:25:07,462 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:07,463 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:07,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:25:07,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T21:25:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:07,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224367619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224367619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224367619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:07,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T21:25:07,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:07,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:07,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:07,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:25:07,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/8945b366dccc40208ca5a1189a10e5e3 2024-11-21T21:25:07,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/84e5ff25421c40efb68c8f7a3c13fd53 is 50, key is test_row_0/C:col10/1732224305842/Put/seqid=0 2024-11-21T21:25:07,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742118_1294 (size=12151) 2024-11-21T21:25:07,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T21:25:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:07,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:07,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:07,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224367999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224368000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:08,001 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in d6ccac84b1b168c8055162a63f397098 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,002 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in d6ccac84b1b168c8055162a63f397098 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:08,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T21:25:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:08,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:08,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:25:08,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224368130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:08,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224368130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224368130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:08,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/84e5ff25421c40efb68c8f7a3c13fd53 2024-11-21T21:25:08,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/60b3fdfb0ff240c4a0292985eb47cd27 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/60b3fdfb0ff240c4a0292985eb47cd27 2024-11-21T21:25:08,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/60b3fdfb0ff240c4a0292985eb47cd27, entries=250, sequenceid=159, filesize=16.5 K 2024-11-21T21:25:08,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/8945b366dccc40208ca5a1189a10e5e3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/8945b366dccc40208ca5a1189a10e5e3 2024-11-21T21:25:08,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T21:25:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:08,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/8945b366dccc40208ca5a1189a10e5e3, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T21:25:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/84e5ff25421c40efb68c8f7a3c13fd53 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/84e5ff25421c40efb68c8f7a3c13fd53 2024-11-21T21:25:08,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/84e5ff25421c40efb68c8f7a3c13fd53, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T21:25:08,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for d6ccac84b1b168c8055162a63f397098 in 1263ms, sequenceid=159, compaction requested=true 2024-11-21T21:25:08,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:08,246 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:08,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:08,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:08,247 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:08,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:08,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:08,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:08,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:08,248 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:08,248 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:08,248 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,248 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:08,248 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:08,248 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/541e79c11aa240b48112f26379d76812, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bf4a883379d947b08099faf0d12d627b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/60b3fdfb0ff240c4a0292985eb47cd27] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=40.2 K 2024-11-21T21:25:08,248 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,248 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/fb25b3aa80f44de29f9896c9b6ba9e46, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1fa1aee16d6c429ca09171ca8aeeb425, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/8945b366dccc40208ca5a1189a10e5e3] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=35.5 K 2024-11-21T21:25:08,249 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541e79c11aa240b48112f26379d76812, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732224304344 2024-11-21T21:25:08,249 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting fb25b3aa80f44de29f9896c9b6ba9e46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732224304344 2024-11-21T21:25:08,250 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa1aee16d6c429ca09171ca8aeeb425, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732224304680 2024-11-21T21:25:08,250 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf4a883379d947b08099faf0d12d627b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732224304680 2024-11-21T21:25:08,250 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8945b366dccc40208ca5a1189a10e5e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732224305842 2024-11-21T21:25:08,250 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60b3fdfb0ff240c4a0292985eb47cd27, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732224305842 2024-11-21T21:25:08,270 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:08,271 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/d8ec24ea209a46d7bb68e3f855f7505b is 50, key is test_row_0/A:col10/1732224305842/Put/seqid=0 2024-11-21T21:25:08,275 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:08,275 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/4e2718261f724fa7bfa4d7902ae96a85 is 50, key is test_row_0/B:col10/1732224305842/Put/seqid=0 2024-11-21T21:25:08,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742119_1295 (size=12493) 2024-11-21T21:25:08,310 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/d8ec24ea209a46d7bb68e3f855f7505b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d8ec24ea209a46d7bb68e3f855f7505b 2024-11-21T21:25:08,317 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into d8ec24ea209a46d7bb68e3f855f7505b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:08,317 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:08,317 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=13, startTime=1732224308245; duration=0sec 2024-11-21T21:25:08,317 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:08,317 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:08,317 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:08,318 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:08,318 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:08,318 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,319 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/40ac7b43f67a454cb6032b0f682d711a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/49b3e9b9a0c348ff9b959ccea9e663fd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/84e5ff25421c40efb68c8f7a3c13fd53] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=35.5 K 2024-11-21T21:25:08,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ac7b43f67a454cb6032b0f682d711a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732224304344 2024-11-21T21:25:08,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b3e9b9a0c348ff9b959ccea9e663fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732224304680 2024-11-21T21:25:08,320 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e5ff25421c40efb68c8f7a3c13fd53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732224305842 2024-11-21T21:25:08,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742120_1296 (size=12493) 2024-11-21T21:25:08,347 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:08,348 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/1f9730ad4f834fb4bfe6037c8fd178a4 is 50, key is test_row_0/C:col10/1732224305842/Put/seqid=0 2024-11-21T21:25:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742121_1297 (size=12493) 2024-11-21T21:25:08,360 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/1f9730ad4f834fb4bfe6037c8fd178a4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/1f9730ad4f834fb4bfe6037c8fd178a4 2024-11-21T21:25:08,378 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into 1f9730ad4f834fb4bfe6037c8fd178a4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:08,378 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:08,378 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=13, startTime=1732224308247; duration=0sec 2024-11-21T21:25:08,378 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:08,378 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:08,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T21:25:08,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:08,383 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T21:25:08,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2e70157496db4b5583fa3c20751c8a1e is 50, key is test_row_1/A:col10/1732224306990/Put/seqid=0 2024-11-21T21:25:08,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742122_1298 (size=9757) 2024-11-21T21:25:08,415 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2e70157496db4b5583fa3c20751c8a1e 2024-11-21T21:25:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/4a32e7c21e0f4627abd82947e0d69158 is 50, key is test_row_1/B:col10/1732224306990/Put/seqid=0 2024-11-21T21:25:08,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742123_1299 (size=9757) 2024-11-21T21:25:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:25:08,740 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/4e2718261f724fa7bfa4d7902ae96a85 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4e2718261f724fa7bfa4d7902ae96a85 2024-11-21T21:25:08,745 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into 4e2718261f724fa7bfa4d7902ae96a85(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:08,745 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:08,745 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=13, startTime=1732224308247; duration=0sec 2024-11-21T21:25:08,745 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:08,745 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:08,835 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/4a32e7c21e0f4627abd82947e0d69158 2024-11-21T21:25:08,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/7752ad8d920b4dab93452eedb6a2f371 is 50, key is test_row_1/C:col10/1732224306990/Put/seqid=0 2024-11-21T21:25:08,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742124_1300 (size=9757) 2024-11-21T21:25:09,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:09,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224369213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224369216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224369217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,252 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/7752ad8d920b4dab93452eedb6a2f371 2024-11-21T21:25:09,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2e70157496db4b5583fa3c20751c8a1e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2e70157496db4b5583fa3c20751c8a1e 2024-11-21T21:25:09,264 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2e70157496db4b5583fa3c20751c8a1e, entries=100, sequenceid=169, filesize=9.5 K 2024-11-21T21:25:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/4a32e7c21e0f4627abd82947e0d69158 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4a32e7c21e0f4627abd82947e0d69158 2024-11-21T21:25:09,269 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4a32e7c21e0f4627abd82947e0d69158, entries=100, sequenceid=169, filesize=9.5 K 2024-11-21T21:25:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/7752ad8d920b4dab93452eedb6a2f371 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7752ad8d920b4dab93452eedb6a2f371 2024-11-21T21:25:09,274 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7752ad8d920b4dab93452eedb6a2f371, entries=100, sequenceid=169, filesize=9.5 K 2024-11-21T21:25:09,275 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for d6ccac84b1b168c8055162a63f397098 in 892ms, sequenceid=169, compaction requested=false 2024-11-21T21:25:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T21:25:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T21:25:09,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T21:25:09,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8130 sec 2024-11-21T21:25:09,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8200 sec 2024-11-21T21:25:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:09,323 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:09,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-21T21:25:09,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/aa24fc2d1576483fb276909d7b4af30f is 50, key is test_row_0/A:col10/1732224309212/Put/seqid=0 2024-11-21T21:25:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224369330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224369333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224369334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742125_1301 (size=14541) 2024-11-21T21:25:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224369440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224369440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224369549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:25:09,609 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:09,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T21:25:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:25:09,612 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:09,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:09,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:09,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224369650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224369650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:25:09,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/aa24fc2d1576483fb276909d7b4af30f 2024-11-21T21:25:09,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/934cc046ea394b5cae951c3358c0b53f is 50, key is test_row_0/B:col10/1732224309212/Put/seqid=0 2024-11-21T21:25:09,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742126_1302 (size=12151) 2024-11-21T21:25:09,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/934cc046ea394b5cae951c3358c0b53f 2024-11-21T21:25:09,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:09,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:09,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:09,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:09,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/38621f9167b64b69b861c92d07198c3a is 50, key is test_row_0/C:col10/1732224309212/Put/seqid=0 2024-11-21T21:25:09,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742127_1303 (size=12151) 2024-11-21T21:25:09,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/38621f9167b64b69b861c92d07198c3a 2024-11-21T21:25:09,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/aa24fc2d1576483fb276909d7b4af30f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/aa24fc2d1576483fb276909d7b4af30f 2024-11-21T21:25:09,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/aa24fc2d1576483fb276909d7b4af30f, entries=200, sequenceid=200, filesize=14.2 K 2024-11-21T21:25:09,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/934cc046ea394b5cae951c3358c0b53f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/934cc046ea394b5cae951c3358c0b53f 2024-11-21T21:25:09,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/934cc046ea394b5cae951c3358c0b53f, entries=150, sequenceid=200, filesize=11.9 K 2024-11-21T21:25:09,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/38621f9167b64b69b861c92d07198c3a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/38621f9167b64b69b861c92d07198c3a 2024-11-21T21:25:09,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/38621f9167b64b69b861c92d07198c3a, entries=150, sequenceid=200, filesize=11.9 K 2024-11-21T21:25:09,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for d6ccac84b1b168c8055162a63f397098 in 509ms, sequenceid=200, compaction requested=true 2024-11-21T21:25:09,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:09,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:09,833 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:09,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:09,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:09,833 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:09,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:09,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:09,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:09,834 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:09,834 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:09,835 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:09,835 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:09,835 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,835 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,835 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d8ec24ea209a46d7bb68e3f855f7505b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2e70157496db4b5583fa3c20751c8a1e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/aa24fc2d1576483fb276909d7b4af30f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=35.9 K 2024-11-21T21:25:09,835 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4e2718261f724fa7bfa4d7902ae96a85, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4a32e7c21e0f4627abd82947e0d69158, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/934cc046ea394b5cae951c3358c0b53f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=33.6 K 2024-11-21T21:25:09,835 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e2718261f724fa7bfa4d7902ae96a85, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732224305842 2024-11-21T21:25:09,835 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8ec24ea209a46d7bb68e3f855f7505b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732224305842 2024-11-21T21:25:09,835 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a32e7c21e0f4627abd82947e0d69158, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732224306988 2024-11-21T21:25:09,835 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e70157496db4b5583fa3c20751c8a1e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732224306988 2024-11-21T21:25:09,836 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa24fc2d1576483fb276909d7b4af30f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732224309212 2024-11-21T21:25:09,836 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 934cc046ea394b5cae951c3358c0b53f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732224309212 2024-11-21T21:25:09,850 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:09,851 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e3fb8f2b352b45ff99e6b270d98efaf6 is 50, key is test_row_0/A:col10/1732224309212/Put/seqid=0 2024-11-21T21:25:09,870 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:09,871 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/1bdeab71875c4933922390916abcb06a is 50, key is test_row_0/B:col10/1732224309212/Put/seqid=0 2024-11-21T21:25:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:09,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:09,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:25:09,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/bd26926b053040258de7f395b5de52c6 is 50, key is test_row_0/A:col10/1732224309872/Put/seqid=0 2024-11-21T21:25:09,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742128_1304 (size=12595) 2024-11-21T21:25:09,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742129_1305 (size=12595) 2024-11-21T21:25:09,916 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/1bdeab71875c4933922390916abcb06a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1bdeab71875c4933922390916abcb06a 2024-11-21T21:25:09,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742130_1306 (size=9757) 2024-11-21T21:25:09,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/bd26926b053040258de7f395b5de52c6 2024-11-21T21:25:09,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:09,924 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into 1bdeab71875c4933922390916abcb06a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:09,924 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:09,924 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=13, startTime=1732224309833; duration=0sec 2024-11-21T21:25:09,924 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:09,924 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:09,924 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:09,927 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:09,927 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:09,928 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:09,928 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/1f9730ad4f834fb4bfe6037c8fd178a4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7752ad8d920b4dab93452eedb6a2f371, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/38621f9167b64b69b861c92d07198c3a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=33.6 K 2024-11-21T21:25:09,929 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f9730ad4f834fb4bfe6037c8fd178a4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732224305842 2024-11-21T21:25:09,929 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7752ad8d920b4dab93452eedb6a2f371, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732224306988 2024-11-21T21:25:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:25:09,930 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 38621f9167b64b69b861c92d07198c3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732224309212 2024-11-21T21:25:09,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/e73dc7b552c647368e4cbbdd492bd5f8 is 50, key is test_row_0/B:col10/1732224309872/Put/seqid=0 2024-11-21T21:25:09,950 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:09,951 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ad839f1b925f41969b115d2d31c75fe2 is 50, key is test_row_0/C:col10/1732224309212/Put/seqid=0 2024-11-21T21:25:09,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224369973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224369977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224369980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:09,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742131_1307 (size=9757) 2024-11-21T21:25:10,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742132_1308 (size=12595) 2024-11-21T21:25:10,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224370020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,023 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in d6ccac84b1b168c8055162a63f397098 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224370029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,032 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in d6ccac84b1b168c8055162a63f397098 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224370090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224370090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224370090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:10,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:10,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:25:10,297 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e3fb8f2b352b45ff99e6b270d98efaf6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e3fb8f2b352b45ff99e6b270d98efaf6 2024-11-21T21:25:10,302 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into e3fb8f2b352b45ff99e6b270d98efaf6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:10,302 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:10,302 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=13, startTime=1732224309833; duration=0sec 2024-11-21T21:25:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,302 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224370300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224370300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,303 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224370309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:10,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:10,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/e73dc7b552c647368e4cbbdd492bd5f8 2024-11-21T21:25:10,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ae4e81af195c45dba29b337eb3abfe37 is 50, key is test_row_0/C:col10/1732224309872/Put/seqid=0 2024-11-21T21:25:10,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742133_1309 (size=9757) 2024-11-21T21:25:10,413 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ad839f1b925f41969b115d2d31c75fe2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ad839f1b925f41969b115d2d31c75fe2 2024-11-21T21:25:10,417 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into ad839f1b925f41969b115d2d31c75fe2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:10,417 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:10,417 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=13, startTime=1732224309833; duration=0sec 2024-11-21T21:25:10,417 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:10,417 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:10,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:10,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:10,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224370609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224370609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224370620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:10,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:25:10,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ae4e81af195c45dba29b337eb3abfe37 2024-11-21T21:25:10,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/bd26926b053040258de7f395b5de52c6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bd26926b053040258de7f395b5de52c6 2024-11-21T21:25:10,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bd26926b053040258de7f395b5de52c6, entries=100, sequenceid=211, filesize=9.5 K 2024-11-21T21:25:10,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/e73dc7b552c647368e4cbbdd492bd5f8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/e73dc7b552c647368e4cbbdd492bd5f8 2024-11-21T21:25:10,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/e73dc7b552c647368e4cbbdd492bd5f8, entries=100, sequenceid=211, filesize=9.5 K 2024-11-21T21:25:10,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/ae4e81af195c45dba29b337eb3abfe37 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ae4e81af195c45dba29b337eb3abfe37 2024-11-21T21:25:10,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ae4e81af195c45dba29b337eb3abfe37, entries=100, sequenceid=211, filesize=9.5 K 2024-11-21T21:25:10,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d6ccac84b1b168c8055162a63f397098 in 944ms, sequenceid=211, compaction requested=false 2024-11-21T21:25:10,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:10,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:25:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:10,843 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:25:10,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/d68f40d2f120424a9f77c3bd2b0aff53 is 50, key is test_row_0/A:col10/1732224309977/Put/seqid=0 2024-11-21T21:25:10,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742134_1310 (size=12151) 2024-11-21T21:25:10,852 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/d68f40d2f120424a9f77c3bd2b0aff53 2024-11-21T21:25:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/776537fe4d5347379375538d980e47f6 is 50, key is test_row_0/B:col10/1732224309977/Put/seqid=0 2024-11-21T21:25:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742135_1311 (size=12151) 2024-11-21T21:25:10,886 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/776537fe4d5347379375538d980e47f6 2024-11-21T21:25:10,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/9cf370b9f206499d9131e849d72d6ea5 is 50, key is test_row_0/C:col10/1732224309977/Put/seqid=0 2024-11-21T21:25:10,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742136_1312 (size=12151) 2024-11-21T21:25:11,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:11,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224371134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224371134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224371136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224371239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224371249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224371249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,302 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/9cf370b9f206499d9131e849d72d6ea5 2024-11-21T21:25:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/d68f40d2f120424a9f77c3bd2b0aff53 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d68f40d2f120424a9f77c3bd2b0aff53 2024-11-21T21:25:11,310 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d68f40d2f120424a9f77c3bd2b0aff53, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T21:25:11,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/776537fe4d5347379375538d980e47f6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/776537fe4d5347379375538d980e47f6 2024-11-21T21:25:11,314 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/776537fe4d5347379375538d980e47f6, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T21:25:11,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/9cf370b9f206499d9131e849d72d6ea5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9cf370b9f206499d9131e849d72d6ea5 2024-11-21T21:25:11,319 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9cf370b9f206499d9131e849d72d6ea5, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T21:25:11,320 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d6ccac84b1b168c8055162a63f397098 in 477ms, sequenceid=239, compaction requested=true 2024-11-21T21:25:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T21:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T21:25:11,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T21:25:11,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7080 sec 2024-11-21T21:25:11,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7140 sec 2024-11-21T21:25:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:11,452 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:11,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:25:11,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0a463e0d09794dfc863ab76e165078e0 is 50, key is test_row_0/A:col10/1732224311132/Put/seqid=0 2024-11-21T21:25:11,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742137_1313 (size=14541) 2024-11-21T21:25:11,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0a463e0d09794dfc863ab76e165078e0 2024-11-21T21:25:11,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/a1b4c587d6fe40ff9ce2106ccf676e2e is 50, key is test_row_0/B:col10/1732224311132/Put/seqid=0 2024-11-21T21:25:11,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742138_1314 (size=12151) 2024-11-21T21:25:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224371520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224371520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224371526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224371630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224371630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224371640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:25:11,760 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:11,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T21:25:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:25:11,762 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:11,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:11,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:11,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224371839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224371840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224371850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:25:11,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/a1b4c587d6fe40ff9ce2106ccf676e2e 2024-11-21T21:25:11,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/da4c0eb3f3a6453a92389e33a4cbf0dd is 50, key is test_row_0/C:col10/1732224311132/Put/seqid=0 2024-11-21T21:25:11,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742139_1315 (size=12151) 2024-11-21T21:25:11,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T21:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:11,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:12,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T21:25:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:25:12,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224372149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224372149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224372159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T21:25:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:12,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:12,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:12,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/da4c0eb3f3a6453a92389e33a4cbf0dd 2024-11-21T21:25:12,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0a463e0d09794dfc863ab76e165078e0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0a463e0d09794dfc863ab76e165078e0 2024-11-21T21:25:12,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0a463e0d09794dfc863ab76e165078e0, entries=200, sequenceid=251, filesize=14.2 K 2024-11-21T21:25:12,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/a1b4c587d6fe40ff9ce2106ccf676e2e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/a1b4c587d6fe40ff9ce2106ccf676e2e 2024-11-21T21:25:12,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/a1b4c587d6fe40ff9ce2106ccf676e2e, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T21:25:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/da4c0eb3f3a6453a92389e33a4cbf0dd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/da4c0eb3f3a6453a92389e33a4cbf0dd 2024-11-21T21:25:12,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/da4c0eb3f3a6453a92389e33a4cbf0dd, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T21:25:12,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d6ccac84b1b168c8055162a63f397098 in 853ms, sequenceid=251, compaction requested=true 2024-11-21T21:25:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:12,306 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:12,306 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:12,307 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49044 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:12,307 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:12,307 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,307 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e3fb8f2b352b45ff99e6b270d98efaf6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bd26926b053040258de7f395b5de52c6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d68f40d2f120424a9f77c3bd2b0aff53, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0a463e0d09794dfc863ab76e165078e0] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=47.9 K 2024-11-21T21:25:12,307 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:12,307 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:12,307 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,308 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1bdeab71875c4933922390916abcb06a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/e73dc7b552c647368e4cbbdd492bd5f8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/776537fe4d5347379375538d980e47f6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/a1b4c587d6fe40ff9ce2106ccf676e2e] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=45.6 K 2024-11-21T21:25:12,308 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3fb8f2b352b45ff99e6b270d98efaf6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732224309212 2024-11-21T21:25:12,308 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bdeab71875c4933922390916abcb06a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732224309212 2024-11-21T21:25:12,308 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e73dc7b552c647368e4cbbdd492bd5f8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732224309872 2024-11-21T21:25:12,308 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd26926b053040258de7f395b5de52c6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732224309872 2024-11-21T21:25:12,309 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 776537fe4d5347379375538d980e47f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732224309965 2024-11-21T21:25:12,309 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d68f40d2f120424a9f77c3bd2b0aff53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732224309965 2024-11-21T21:25:12,309 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a1b4c587d6fe40ff9ce2106ccf676e2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224311122 2024-11-21T21:25:12,309 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a463e0d09794dfc863ab76e165078e0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224311122 2024-11-21T21:25:12,318 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:12,318 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/c7de4ab5701e4bbca0839426b273ac54 is 50, key is test_row_0/B:col10/1732224311132/Put/seqid=0 2024-11-21T21:25:12,318 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:12,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/7e3c18c25c3a4a29b69615c7401a6ffc is 50, key is test_row_0/A:col10/1732224311132/Put/seqid=0 2024-11-21T21:25:12,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742140_1316 (size=12731) 2024-11-21T21:25:12,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742141_1317 (size=12731) 2024-11-21T21:25:12,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T21:25:12,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,373 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:25:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3e19052fdff0423bbaf74be30f2d4c35 is 50, key is test_row_0/A:col10/1732224311490/Put/seqid=0 2024-11-21T21:25:12,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742142_1318 (size=12301) 2024-11-21T21:25:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:25:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:12,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224372674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224372675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224372679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,735 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/c7de4ab5701e4bbca0839426b273ac54 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/c7de4ab5701e4bbca0839426b273ac54 2024-11-21T21:25:12,738 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/7e3c18c25c3a4a29b69615c7401a6ffc as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/7e3c18c25c3a4a29b69615c7401a6ffc 2024-11-21T21:25:12,738 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into c7de4ab5701e4bbca0839426b273ac54(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:12,739 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:12,739 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=12, startTime=1732224312306; duration=0sec 2024-11-21T21:25:12,739 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:12,739 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:12,739 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:12,740 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:12,740 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:12,740 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:12,740 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ad839f1b925f41969b115d2d31c75fe2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ae4e81af195c45dba29b337eb3abfe37, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9cf370b9f206499d9131e849d72d6ea5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/da4c0eb3f3a6453a92389e33a4cbf0dd] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=45.6 K 2024-11-21T21:25:12,741 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ad839f1b925f41969b115d2d31c75fe2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732224309212 2024-11-21T21:25:12,741 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4e81af195c45dba29b337eb3abfe37, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732224309872 2024-11-21T21:25:12,741 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf370b9f206499d9131e849d72d6ea5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732224309965 2024-11-21T21:25:12,741 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting da4c0eb3f3a6453a92389e33a4cbf0dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224311122 2024-11-21T21:25:12,743 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into 7e3c18c25c3a4a29b69615c7401a6ffc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:12,743 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:12,743 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=12, startTime=1732224312306; duration=0sec 2024-11-21T21:25:12,743 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:12,743 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:12,750 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:12,750 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/e7129e2042314905872077b1a82fad94 is 50, key is test_row_0/C:col10/1732224311132/Put/seqid=0 2024-11-21T21:25:12,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742143_1319 (size=12731) 2024-11-21T21:25:12,782 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3e19052fdff0423bbaf74be30f2d4c35 2024-11-21T21:25:12,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/dbd0c910da954cf998c003cd80d10321 is 50, key is test_row_0/B:col10/1732224311490/Put/seqid=0 2024-11-21T21:25:12,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742144_1320 (size=12301) 2024-11-21T21:25:12,792 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/dbd0c910da954cf998c003cd80d10321 2024-11-21T21:25:12,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224372789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224372789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224372789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:12,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/65b3dbeb16ce4d2aa0852e7642c7f0cb is 50, key is test_row_0/C:col10/1732224311490/Put/seqid=0 2024-11-21T21:25:12,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742145_1321 (size=12301) 2024-11-21T21:25:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:25:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224373000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224373000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224373000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,158 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/e7129e2042314905872077b1a82fad94 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/e7129e2042314905872077b1a82fad94 2024-11-21T21:25:13,163 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into e7129e2042314905872077b1a82fad94(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:13,163 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:13,163 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=12, startTime=1732224312306; duration=0sec 2024-11-21T21:25:13,163 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:13,163 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/65b3dbeb16ce4d2aa0852e7642c7f0cb 2024-11-21T21:25:13,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3e19052fdff0423bbaf74be30f2d4c35 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3e19052fdff0423bbaf74be30f2d4c35 2024-11-21T21:25:13,209 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3e19052fdff0423bbaf74be30f2d4c35, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T21:25:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/dbd0c910da954cf998c003cd80d10321 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/dbd0c910da954cf998c003cd80d10321 2024-11-21T21:25:13,213 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/dbd0c910da954cf998c003cd80d10321, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T21:25:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/65b3dbeb16ce4d2aa0852e7642c7f0cb as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/65b3dbeb16ce4d2aa0852e7642c7f0cb 2024-11-21T21:25:13,217 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/65b3dbeb16ce4d2aa0852e7642c7f0cb, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T21:25:13,217 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d6ccac84b1b168c8055162a63f397098 in 844ms, sequenceid=275, compaction requested=false 2024-11-21T21:25:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T21:25:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T21:25:13,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T21:25:13,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4560 sec 2024-11-21T21:25:13,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4600 sec 2024-11-21T21:25:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:13,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:13,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:25:13,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e381f41ce3104cd1b1d2879b599bb6ff is 50, key is test_row_0/A:col10/1732224312665/Put/seqid=0 2024-11-21T21:25:13,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742146_1322 (size=14741) 2024-11-21T21:25:13,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e381f41ce3104cd1b1d2879b599bb6ff 2024-11-21T21:25:13,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/af84e6f4065b4f98b9a0bed69ef1b63b is 50, key is test_row_0/B:col10/1732224312665/Put/seqid=0 2024-11-21T21:25:13,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742147_1323 (size=12301) 2024-11-21T21:25:13,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224373337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224373341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224373342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224373450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224373450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224373450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224373659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224373659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224373660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/af84e6f4065b4f98b9a0bed69ef1b63b 2024-11-21T21:25:13,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/6880078e31284fa5bb32510a7f7c9d59 is 50, key is test_row_0/C:col10/1732224312665/Put/seqid=0 2024-11-21T21:25:13,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742148_1324 (size=12301) 2024-11-21T21:25:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:25:13,910 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:13,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T21:25:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:25:13,912 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:13,912 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:13,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224373969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224373969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224373969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:25:14,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:25:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:14,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:14,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:14,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732224374070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42804 deadline: 1732224374070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,073 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in d6ccac84b1b168c8055162a63f397098 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,073 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in d6ccac84b1b168c8055162a63f397098 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/6880078e31284fa5bb32510a7f7c9d59 2024-11-21T21:25:14,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e381f41ce3104cd1b1d2879b599bb6ff as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e381f41ce3104cd1b1d2879b599bb6ff 2024-11-21T21:25:14,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e381f41ce3104cd1b1d2879b599bb6ff, entries=200, sequenceid=293, filesize=14.4 K 2024-11-21T21:25:14,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/af84e6f4065b4f98b9a0bed69ef1b63b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/af84e6f4065b4f98b9a0bed69ef1b63b 2024-11-21T21:25:14,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/af84e6f4065b4f98b9a0bed69ef1b63b, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T21:25:14,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/6880078e31284fa5bb32510a7f7c9d59 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/6880078e31284fa5bb32510a7f7c9d59 2024-11-21T21:25:14,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/6880078e31284fa5bb32510a7f7c9d59, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T21:25:14,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d6ccac84b1b168c8055162a63f397098 in 844ms, sequenceid=293, compaction requested=true 2024-11-21T21:25:14,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:14,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:14,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:14,155 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:14,155 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:14,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:14,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:14,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:14,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:14,156 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:14,156 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:14,156 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:14,156 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:14,157 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:14,157 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:14,157 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/c7de4ab5701e4bbca0839426b273ac54, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/dbd0c910da954cf998c003cd80d10321, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/af84e6f4065b4f98b9a0bed69ef1b63b] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=36.5 K 2024-11-21T21:25:14,157 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/7e3c18c25c3a4a29b69615c7401a6ffc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3e19052fdff0423bbaf74be30f2d4c35, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e381f41ce3104cd1b1d2879b599bb6ff] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=38.8 K 2024-11-21T21:25:14,157 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c7de4ab5701e4bbca0839426b273ac54, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224311122 2024-11-21T21:25:14,157 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e3c18c25c3a4a29b69615c7401a6ffc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224311122 2024-11-21T21:25:14,157 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting dbd0c910da954cf998c003cd80d10321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732224311490 2024-11-21T21:25:14,157 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e19052fdff0423bbaf74be30f2d4c35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732224311490 2024-11-21T21:25:14,157 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting af84e6f4065b4f98b9a0bed69ef1b63b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224312665 2024-11-21T21:25:14,157 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e381f41ce3104cd1b1d2879b599bb6ff, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224312665 2024-11-21T21:25:14,163 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:14,163 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/15e8ddbb07024e938b8dc5d3f96c2e15 is 50, key is test_row_0/B:col10/1732224312665/Put/seqid=0 2024-11-21T21:25:14,164 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:14,164 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/082a58cc04c74a18a6199477d73e7084 is 50, key is test_row_0/A:col10/1732224312665/Put/seqid=0 2024-11-21T21:25:14,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742149_1325 (size=12983) 2024-11-21T21:25:14,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742150_1326 (size=12983) 2024-11-21T21:25:14,177 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/082a58cc04c74a18a6199477d73e7084 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/082a58cc04c74a18a6199477d73e7084 2024-11-21T21:25:14,182 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into 082a58cc04c74a18a6199477d73e7084(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:14,182 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:14,182 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=13, startTime=1732224314155; duration=0sec 2024-11-21T21:25:14,182 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:14,182 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:14,182 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:14,183 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:14,183 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:14,183 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:14,183 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/e7129e2042314905872077b1a82fad94, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/65b3dbeb16ce4d2aa0852e7642c7f0cb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/6880078e31284fa5bb32510a7f7c9d59] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=36.5 K 2024-11-21T21:25:14,183 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7129e2042314905872077b1a82fad94, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732224311122 2024-11-21T21:25:14,184 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b3dbeb16ce4d2aa0852e7642c7f0cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732224311490 2024-11-21T21:25:14,184 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6880078e31284fa5bb32510a7f7c9d59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224312665 2024-11-21T21:25:14,191 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:14,192 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/d3d6276c41a44aa08c24e1a2731ae09a is 50, key is test_row_0/C:col10/1732224312665/Put/seqid=0 2024-11-21T21:25:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742151_1327 (size=12983) 2024-11-21T21:25:14,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:25:14,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:14,218 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:25:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2bfbc404d9e4424f871b0094e001732f is 50, key is test_row_0/A:col10/1732224313340/Put/seqid=0 2024-11-21T21:25:14,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742152_1328 (size=12301) 2024-11-21T21:25:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:25:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:14,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224374496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224374497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224374498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:25:14,571 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/15e8ddbb07024e938b8dc5d3f96c2e15 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/15e8ddbb07024e938b8dc5d3f96c2e15 2024-11-21T21:25:14,576 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into 15e8ddbb07024e938b8dc5d3f96c2e15(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:14,576 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:14,576 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=13, startTime=1732224314155; duration=0sec 2024-11-21T21:25:14,576 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:14,576 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:14,600 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/d3d6276c41a44aa08c24e1a2731ae09a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/d3d6276c41a44aa08c24e1a2731ae09a 2024-11-21T21:25:14,604 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into d3d6276c41a44aa08c24e1a2731ae09a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:14,604 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:14,604 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=13, startTime=1732224314155; duration=0sec 2024-11-21T21:25:14,605 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:14,605 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224374609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224374609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224374609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,627 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2bfbc404d9e4424f871b0094e001732f 2024-11-21T21:25:14,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/31da16f6b7ad46489c72a57d0cee79c1 is 50, key is test_row_0/B:col10/1732224313340/Put/seqid=0 2024-11-21T21:25:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742153_1329 (size=12301) 2024-11-21T21:25:14,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224374820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224374820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224374820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,037 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/31da16f6b7ad46489c72a57d0cee79c1 2024-11-21T21:25:15,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/962d618e03e045268bdeba2fd86e09ff is 50, key is test_row_0/C:col10/1732224313340/Put/seqid=0 2024-11-21T21:25:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742154_1330 (size=12301) 2024-11-21T21:25:15,046 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/962d618e03e045268bdeba2fd86e09ff 2024-11-21T21:25:15,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2bfbc404d9e4424f871b0094e001732f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2bfbc404d9e4424f871b0094e001732f 2024-11-21T21:25:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:25:15,052 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2bfbc404d9e4424f871b0094e001732f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-21T21:25:15,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/31da16f6b7ad46489c72a57d0cee79c1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/31da16f6b7ad46489c72a57d0cee79c1 2024-11-21T21:25:15,057 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/31da16f6b7ad46489c72a57d0cee79c1, entries=150, sequenceid=315, filesize=12.0 K 2024-11-21T21:25:15,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/962d618e03e045268bdeba2fd86e09ff as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/962d618e03e045268bdeba2fd86e09ff 2024-11-21T21:25:15,064 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/962d618e03e045268bdeba2fd86e09ff, entries=150, sequenceid=315, filesize=12.0 K 2024-11-21T21:25:15,065 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d6ccac84b1b168c8055162a63f397098 in 848ms, sequenceid=315, compaction requested=false 2024-11-21T21:25:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T21:25:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T21:25:15,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T21:25:15,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-11-21T21:25:15,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.1590 sec 2024-11-21T21:25:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:15,132 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:15,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T21:25:15,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0e966588e8354e1ebcd30265c0caccfa is 50, key is test_row_0/A:col10/1732224315131/Put/seqid=0 2024-11-21T21:25:15,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742155_1331 (size=17181) 2024-11-21T21:25:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224375154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224375157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224375158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224375269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224375269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224375269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224375479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224375479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224375479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0e966588e8354e1ebcd30265c0caccfa 2024-11-21T21:25:15,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3fee5465eb5d40f89fe282ce9ea0ceef is 50, key is test_row_0/B:col10/1732224315131/Put/seqid=0 2024-11-21T21:25:15,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742156_1332 (size=12301) 2024-11-21T21:25:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224375789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224375789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224375789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:15,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3fee5465eb5d40f89fe282ce9ea0ceef 2024-11-21T21:25:15,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/153af8f0d87447d28249ff0ab63fce94 is 50, key is test_row_0/C:col10/1732224315131/Put/seqid=0 2024-11-21T21:25:15,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742157_1333 (size=12301) 2024-11-21T21:25:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:25:16,060 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:16,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T21:25:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:25:16,062 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:16,062 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:16,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:25:16,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:25:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:16,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224376299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:16,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224376299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:16,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224376299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:16,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:25:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:16,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/153af8f0d87447d28249ff0ab63fce94 2024-11-21T21:25:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:25:16,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0e966588e8354e1ebcd30265c0caccfa as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0e966588e8354e1ebcd30265c0caccfa 2024-11-21T21:25:16,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0e966588e8354e1ebcd30265c0caccfa, entries=250, sequenceid=335, filesize=16.8 K 2024-11-21T21:25:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3fee5465eb5d40f89fe282ce9ea0ceef as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3fee5465eb5d40f89fe282ce9ea0ceef 2024-11-21T21:25:16,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3fee5465eb5d40f89fe282ce9ea0ceef, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T21:25:16,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/153af8f0d87447d28249ff0ab63fce94 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/153af8f0d87447d28249ff0ab63fce94 2024-11-21T21:25:16,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/153af8f0d87447d28249ff0ab63fce94, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T21:25:16,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d6ccac84b1b168c8055162a63f397098 in 1262ms, sequenceid=335, compaction requested=true 2024-11-21T21:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:16,394 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:16,394 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:16,395 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:16,395 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:16,395 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:16,395 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:16,395 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,395 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,395 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/15e8ddbb07024e938b8dc5d3f96c2e15, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/31da16f6b7ad46489c72a57d0cee79c1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3fee5465eb5d40f89fe282ce9ea0ceef] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=36.7 K 2024-11-21T21:25:16,395 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/082a58cc04c74a18a6199477d73e7084, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2bfbc404d9e4424f871b0094e001732f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0e966588e8354e1ebcd30265c0caccfa] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=41.5 K 2024-11-21T21:25:16,395 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 082a58cc04c74a18a6199477d73e7084, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224312665 2024-11-21T21:25:16,395 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 15e8ddbb07024e938b8dc5d3f96c2e15, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224312665 2024-11-21T21:25:16,396 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bfbc404d9e4424f871b0094e001732f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732224313331 2024-11-21T21:25:16,396 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 31da16f6b7ad46489c72a57d0cee79c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732224313331 2024-11-21T21:25:16,396 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fee5465eb5d40f89fe282ce9ea0ceef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732224315131 2024-11-21T21:25:16,396 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e966588e8354e1ebcd30265c0caccfa, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732224314485 2024-11-21T21:25:16,402 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:16,402 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:16,403 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/a150f62d3c6945c584583641b0cd58fe is 50, key is test_row_0/A:col10/1732224315131/Put/seqid=0 2024-11-21T21:25:16,404 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/09d3b4e9769146ff9930f42f002a6664 is 50, key is test_row_0/B:col10/1732224315131/Put/seqid=0 2024-11-21T21:25:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742158_1334 (size=13085) 2024-11-21T21:25:16,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742159_1335 (size=13085) 2024-11-21T21:25:16,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:25:16,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,520 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T21:25:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2960bf2a2466438191ee74ed8a4415f2 is 50, key is test_row_0/A:col10/1732224315157/Put/seqid=0 2024-11-21T21:25:16,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742160_1336 (size=12301) 2024-11-21T21:25:16,527 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2960bf2a2466438191ee74ed8a4415f2 2024-11-21T21:25:16,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/2c5327a07b144ae2afaf74695aebcb37 is 50, key is test_row_0/B:col10/1732224315157/Put/seqid=0 2024-11-21T21:25:16,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742161_1337 (size=12301) 2024-11-21T21:25:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:25:16,823 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/a150f62d3c6945c584583641b0cd58fe as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/a150f62d3c6945c584583641b0cd58fe 2024-11-21T21:25:16,830 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/09d3b4e9769146ff9930f42f002a6664 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/09d3b4e9769146ff9930f42f002a6664 2024-11-21T21:25:16,834 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into a150f62d3c6945c584583641b0cd58fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:16,834 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:16,834 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=13, startTime=1732224316394; duration=0sec 2024-11-21T21:25:16,834 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:16,834 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:16,834 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:16,835 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into 09d3b4e9769146ff9930f42f002a6664(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:16,835 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:16,835 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=13, startTime=1732224316394; duration=0sec 2024-11-21T21:25:16,835 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:16,835 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:16,835 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:16,835 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:16,835 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:16,835 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/d3d6276c41a44aa08c24e1a2731ae09a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/962d618e03e045268bdeba2fd86e09ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/153af8f0d87447d28249ff0ab63fce94] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=36.7 K 2024-11-21T21:25:16,836 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d6276c41a44aa08c24e1a2731ae09a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224312665 2024-11-21T21:25:16,836 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962d618e03e045268bdeba2fd86e09ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732224313331 2024-11-21T21:25:16,837 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 153af8f0d87447d28249ff0ab63fce94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732224315131 2024-11-21T21:25:16,846 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:16,847 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/28150471c62143ba8f4c539b67f20d6d is 50, key is test_row_0/C:col10/1732224315131/Put/seqid=0 2024-11-21T21:25:16,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742162_1338 (size=13085) 2024-11-21T21:25:16,938 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/2c5327a07b144ae2afaf74695aebcb37 2024-11-21T21:25:16,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bae969c73bed4462a7cc5f33c4a7f643 is 50, key is test_row_0/C:col10/1732224315157/Put/seqid=0 2024-11-21T21:25:16,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742163_1339 (size=12301) 2024-11-21T21:25:17,139 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:25:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:25:17,256 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/28150471c62143ba8f4c539b67f20d6d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/28150471c62143ba8f4c539b67f20d6d 2024-11-21T21:25:17,260 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into 28150471c62143ba8f4c539b67f20d6d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:17,260 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:17,260 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=13, startTime=1732224316394; duration=0sec 2024-11-21T21:25:17,260 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:17,260 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:17,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:17,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224377329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224377331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224377333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,347 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bae969c73bed4462a7cc5f33c4a7f643 2024-11-21T21:25:17,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/2960bf2a2466438191ee74ed8a4415f2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2960bf2a2466438191ee74ed8a4415f2 2024-11-21T21:25:17,354 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2960bf2a2466438191ee74ed8a4415f2, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T21:25:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/2c5327a07b144ae2afaf74695aebcb37 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/2c5327a07b144ae2afaf74695aebcb37 2024-11-21T21:25:17,359 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/2c5327a07b144ae2afaf74695aebcb37, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T21:25:17,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bae969c73bed4462a7cc5f33c4a7f643 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bae969c73bed4462a7cc5f33c4a7f643 2024-11-21T21:25:17,363 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bae969c73bed4462a7cc5f33c4a7f643, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T21:25:17,364 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d6ccac84b1b168c8055162a63f397098 in 844ms, sequenceid=353, compaction requested=false 2024-11-21T21:25:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T21:25:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T21:25:17,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T21:25:17,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3030 sec 2024-11-21T21:25:17,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3070 sec 2024-11-21T21:25:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:17,441 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:17,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T21:25:17,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/728a52d617ad4af2a662ed91389382b8 is 50, key is test_row_0/A:col10/1732224317327/Put/seqid=0 2024-11-21T21:25:17,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742164_1340 (size=17181) 2024-11-21T21:25:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224377460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224377463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224377466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224377569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224377569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224377580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224377779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224377779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224377789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:17,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/728a52d617ad4af2a662ed91389382b8 2024-11-21T21:25:17,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/ea36ce5664ff45c080a956061697fd55 is 50, key is test_row_0/B:col10/1732224317327/Put/seqid=0 2024-11-21T21:25:17,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742165_1341 (size=12301) 2024-11-21T21:25:17,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/ea36ce5664ff45c080a956061697fd55 2024-11-21T21:25:17,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/186814e053ab46eb8936552a7eb28f7a is 50, key is test_row_0/C:col10/1732224317327/Put/seqid=0 2024-11-21T21:25:17,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742166_1342 (size=12301) 2024-11-21T21:25:17,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/186814e053ab46eb8936552a7eb28f7a 2024-11-21T21:25:17,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/728a52d617ad4af2a662ed91389382b8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/728a52d617ad4af2a662ed91389382b8 2024-11-21T21:25:17,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/728a52d617ad4af2a662ed91389382b8, entries=250, sequenceid=376, filesize=16.8 K 2024-11-21T21:25:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/ea36ce5664ff45c080a956061697fd55 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/ea36ce5664ff45c080a956061697fd55 2024-11-21T21:25:17,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/ea36ce5664ff45c080a956061697fd55, entries=150, sequenceid=376, filesize=12.0 K 2024-11-21T21:25:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/186814e053ab46eb8936552a7eb28f7a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/186814e053ab46eb8936552a7eb28f7a 2024-11-21T21:25:17,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/186814e053ab46eb8936552a7eb28f7a, entries=150, sequenceid=376, filesize=12.0 K 2024-11-21T21:25:17,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d6ccac84b1b168c8055162a63f397098 in 453ms, sequenceid=376, compaction requested=true 2024-11-21T21:25:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:17,894 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:17,894 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:17,895 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:17,895 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:17,895 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:17,895 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:17,895 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:17,895 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:17,895 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/09d3b4e9769146ff9930f42f002a6664, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/2c5327a07b144ae2afaf74695aebcb37, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/ea36ce5664ff45c080a956061697fd55] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=36.8 K 2024-11-21T21:25:17,895 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/a150f62d3c6945c584583641b0cd58fe, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2960bf2a2466438191ee74ed8a4415f2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/728a52d617ad4af2a662ed91389382b8] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=41.6 K 2024-11-21T21:25:17,895 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d3b4e9769146ff9930f42f002a6664, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732224315131 2024-11-21T21:25:17,895 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a150f62d3c6945c584583641b0cd58fe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732224315131 2024-11-21T21:25:17,896 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2960bf2a2466438191ee74ed8a4415f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732224315150 2024-11-21T21:25:17,896 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c5327a07b144ae2afaf74695aebcb37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732224315150 2024-11-21T21:25:17,897 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ea36ce5664ff45c080a956061697fd55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732224317327 2024-11-21T21:25:17,897 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 728a52d617ad4af2a662ed91389382b8, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732224317318 2024-11-21T21:25:17,903 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:17,903 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:17,904 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/cca6985fd3fc4e2585ced500e6accdb8 is 50, key is test_row_0/B:col10/1732224317327/Put/seqid=0 2024-11-21T21:25:17,904 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/cece9251367f46fe8d191421f257d273 is 50, key is test_row_0/A:col10/1732224317327/Put/seqid=0 2024-11-21T21:25:17,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742168_1344 (size=13187) 2024-11-21T21:25:17,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742167_1343 (size=13187) 2024-11-21T21:25:17,918 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/cca6985fd3fc4e2585ced500e6accdb8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/cca6985fd3fc4e2585ced500e6accdb8 2024-11-21T21:25:17,926 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into cca6985fd3fc4e2585ced500e6accdb8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:17,926 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:17,926 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=13, startTime=1732224317894; duration=0sec 2024-11-21T21:25:17,926 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:17,926 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:17,926 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:17,928 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:17,928 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:17,928 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:17,928 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/28150471c62143ba8f4c539b67f20d6d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bae969c73bed4462a7cc5f33c4a7f643, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/186814e053ab46eb8936552a7eb28f7a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=36.8 K 2024-11-21T21:25:17,928 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 28150471c62143ba8f4c539b67f20d6d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732224315131 2024-11-21T21:25:17,929 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting bae969c73bed4462a7cc5f33c4a7f643, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732224315150 2024-11-21T21:25:17,929 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 186814e053ab46eb8936552a7eb28f7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732224317327 2024-11-21T21:25:17,937 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:17,937 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/4b2c6a0446e845dfa01206eaeaf456dd is 50, key is test_row_0/C:col10/1732224317327/Put/seqid=0 2024-11-21T21:25:17,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742169_1345 (size=13187) 2024-11-21T21:25:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:18,092 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:18,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T21:25:18,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e9465a0f711a4d62b1c128b8f8f3e5a5 is 50, key is test_row_0/A:col10/1732224318091/Put/seqid=0 2024-11-21T21:25:18,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742170_1346 (size=12301) 2024-11-21T21:25:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224378114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224378116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224378117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:25:18,210 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:18,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-21T21:25:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:25:18,212 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:18,213 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:18,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:18,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224378229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224378229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224378229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,317 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/cece9251367f46fe8d191421f257d273 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/cece9251367f46fe8d191421f257d273 2024-11-21T21:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:25:18,321 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into cece9251367f46fe8d191421f257d273(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:18,321 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:18,321 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=13, startTime=1732224317894; duration=0sec 2024-11-21T21:25:18,321 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:18,321 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:18,347 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/4b2c6a0446e845dfa01206eaeaf456dd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/4b2c6a0446e845dfa01206eaeaf456dd 2024-11-21T21:25:18,351 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into 4b2c6a0446e845dfa01206eaeaf456dd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:18,351 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:18,351 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=13, startTime=1732224317894; duration=0sec 2024-11-21T21:25:18,351 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:18,351 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:18,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224378439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224378439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224378439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e9465a0f711a4d62b1c128b8f8f3e5a5 2024-11-21T21:25:18,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/7c8b7ce6ab704dac8150ffb2fcbacabf is 50, key is test_row_0/B:col10/1732224318091/Put/seqid=0 2024-11-21T21:25:18,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742171_1347 (size=12301) 2024-11-21T21:25:18,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:25:18,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224378750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224378750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224378750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:18,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:25:18,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/7c8b7ce6ab704dac8150ffb2fcbacabf 2024-11-21T21:25:18,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bcface94caab466cb9e747d4482493d0 is 50, key is test_row_0/C:col10/1732224318091/Put/seqid=0 2024-11-21T21:25:18,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742172_1348 (size=12301) 2024-11-21T21:25:18,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:18,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:19,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:19,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:19,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224379259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:19,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224379260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:19,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224379260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:19,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:19,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:19,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:19,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:19,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:19,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bcface94caab466cb9e747d4482493d0 2024-11-21T21:25:19,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/e9465a0f711a4d62b1c128b8f8f3e5a5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e9465a0f711a4d62b1c128b8f8f3e5a5 2024-11-21T21:25:19,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e9465a0f711a4d62b1c128b8f8f3e5a5, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T21:25:19,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/7c8b7ce6ab704dac8150ffb2fcbacabf as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/7c8b7ce6ab704dac8150ffb2fcbacabf 2024-11-21T21:25:19,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/7c8b7ce6ab704dac8150ffb2fcbacabf, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T21:25:19,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bcface94caab466cb9e747d4482493d0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bcface94caab466cb9e747d4482493d0 2024-11-21T21:25:19,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bcface94caab466cb9e747d4482493d0, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T21:25:19,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d6ccac84b1b168c8055162a63f397098 in 1247ms, sequenceid=395, compaction requested=false 2024-11-21T21:25:19,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:25:19,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:25:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:19,435 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T21:25:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/5ac777b72ac34fd2bc328af89d1f71d1 is 50, key is test_row_0/A:col10/1732224318115/Put/seqid=0 2024-11-21T21:25:19,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742173_1349 (size=12301) 2024-11-21T21:25:19,842 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/5ac777b72ac34fd2bc328af89d1f71d1 2024-11-21T21:25:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/79e1e007fd6f4d4690ca953f8700b8ca is 50, key is test_row_0/B:col10/1732224318115/Put/seqid=0 2024-11-21T21:25:19,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742174_1350 (size=12301) 2024-11-21T21:25:20,270 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/79e1e007fd6f4d4690ca953f8700b8ca 2024-11-21T21:25:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:20,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. as already flushing 2024-11-21T21:25:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/91c00dea314e4962bbc65b7e4e5a1428 is 50, key is test_row_0/C:col10/1732224318115/Put/seqid=0 2024-11-21T21:25:20,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742175_1351 (size=12301) 2024-11-21T21:25:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224380294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224380295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224380297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:25:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224380409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224380409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224380409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224380619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224380619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224380619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,686 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/91c00dea314e4962bbc65b7e4e5a1428 2024-11-21T21:25:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/5ac777b72ac34fd2bc328af89d1f71d1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/5ac777b72ac34fd2bc328af89d1f71d1 2024-11-21T21:25:20,692 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/5ac777b72ac34fd2bc328af89d1f71d1, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T21:25:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/79e1e007fd6f4d4690ca953f8700b8ca as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/79e1e007fd6f4d4690ca953f8700b8ca 2024-11-21T21:25:20,696 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/79e1e007fd6f4d4690ca953f8700b8ca, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T21:25:20,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/91c00dea314e4962bbc65b7e4e5a1428 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/91c00dea314e4962bbc65b7e4e5a1428 2024-11-21T21:25:20,705 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/91c00dea314e4962bbc65b7e4e5a1428, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T21:25:20,706 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d6ccac84b1b168c8055162a63f397098 in 1271ms, sequenceid=415, compaction requested=true 2024-11-21T21:25:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-21T21:25:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-21T21:25:20,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T21:25:20,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4940 sec 2024-11-21T21:25:20,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.4990 sec 2024-11-21T21:25:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:20,932 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:20,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T21:25:20,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0d55259659914708ae0995af3c158ce2 is 50, key is test_row_0/A:col10/1732224320930/Put/seqid=0 2024-11-21T21:25:20,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742176_1352 (size=12301) 2024-11-21T21:25:20,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0d55259659914708ae0995af3c158ce2 2024-11-21T21:25:20,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3c935a1731d34058a4d5e44286d22dae is 50, key is test_row_0/B:col10/1732224320930/Put/seqid=0 2024-11-21T21:25:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224380965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224380968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224380968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:20,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742177_1353 (size=12301) 2024-11-21T21:25:20,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3c935a1731d34058a4d5e44286d22dae 2024-11-21T21:25:21,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/9098d42cbeeb4f759c3b29830a5e2d8c is 50, key is test_row_0/C:col10/1732224320930/Put/seqid=0 2024-11-21T21:25:21,045 INFO [Thread-1229 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1229. 2024-11-21T21:25:21,045 DEBUG [Thread-1229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,045 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,045 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,046 INFO [Thread-1231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1231. 2024-11-21T21:25:21,046 DEBUG [Thread-1231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,046 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,046 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742178_1354 (size=12301) 2024-11-21T21:25:21,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,049 INFO [Thread-1235 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1235. 2024-11-21T21:25:21,050 DEBUG [Thread-1235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,050 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,050 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,050 INFO [Thread-1237 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1237. 2024-11-21T21:25:21,050 DEBUG [Thread-1237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,050 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,050 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,054 INFO [Thread-1233 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1233. 2024-11-21T21:25:21,054 DEBUG [Thread-1233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,054 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,054 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224381080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224381080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224381080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42864 deadline: 1732224381290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732224381290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42854 deadline: 1732224381290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:21,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/9098d42cbeeb4f759c3b29830a5e2d8c 2024-11-21T21:25:21,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/0d55259659914708ae0995af3c158ce2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d55259659914708ae0995af3c158ce2 2024-11-21T21:25:21,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d55259659914708ae0995af3c158ce2, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T21:25:21,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/3c935a1731d34058a4d5e44286d22dae as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3c935a1731d34058a4d5e44286d22dae 2024-11-21T21:25:21,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3c935a1731d34058a4d5e44286d22dae, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T21:25:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/9098d42cbeeb4f759c3b29830a5e2d8c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9098d42cbeeb4f759c3b29830a5e2d8c 2024-11-21T21:25:21,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9098d42cbeeb4f759c3b29830a5e2d8c, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T21:25:21,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for d6ccac84b1b168c8055162a63f397098 in 534ms, sequenceid=435, compaction requested=true 2024-11-21T21:25:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:21,466 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:21,466 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ccac84b1b168c8055162a63f397098:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:21,467 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:21,468 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/B is initiating minor compaction (all files) 2024-11-21T21:25:21,468 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/B in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:21,468 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/cca6985fd3fc4e2585ced500e6accdb8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/7c8b7ce6ab704dac8150ffb2fcbacabf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/79e1e007fd6f4d4690ca953f8700b8ca, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3c935a1731d34058a4d5e44286d22dae] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=48.9 K 2024-11-21T21:25:21,468 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:21,468 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/A is initiating minor compaction (all files) 2024-11-21T21:25:21,468 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/A in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:21,468 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/cece9251367f46fe8d191421f257d273, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e9465a0f711a4d62b1c128b8f8f3e5a5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/5ac777b72ac34fd2bc328af89d1f71d1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d55259659914708ae0995af3c158ce2] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=48.9 K 2024-11-21T21:25:21,468 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting cca6985fd3fc4e2585ced500e6accdb8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732224317327 2024-11-21T21:25:21,469 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting cece9251367f46fe8d191421f257d273, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732224317327 2024-11-21T21:25:21,469 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8b7ce6ab704dac8150ffb2fcbacabf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732224317459 2024-11-21T21:25:21,469 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9465a0f711a4d62b1c128b8f8f3e5a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732224317459 2024-11-21T21:25:21,469 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e1e007fd6f4d4690ca953f8700b8ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732224318112 2024-11-21T21:25:21,469 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac777b72ac34fd2bc328af89d1f71d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732224318112 2024-11-21T21:25:21,469 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c935a1731d34058a4d5e44286d22dae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732224320294 2024-11-21T21:25:21,469 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d55259659914708ae0995af3c158ce2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732224320294 2024-11-21T21:25:21,480 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:21,480 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/6b510b2f55644b5bb8d591596b00d80f is 50, key is test_row_0/B:col10/1732224320930/Put/seqid=0 2024-11-21T21:25:21,483 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#A#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:21,483 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/962b27c3fe114ae9b7e8c0ff47035da9 is 50, key is test_row_0/A:col10/1732224320930/Put/seqid=0 2024-11-21T21:25:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742179_1355 (size=13323) 2024-11-21T21:25:21,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742180_1356 (size=13323) 2024-11-21T21:25:21,498 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/6b510b2f55644b5bb8d591596b00d80f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/6b510b2f55644b5bb8d591596b00d80f 2024-11-21T21:25:21,499 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/962b27c3fe114ae9b7e8c0ff47035da9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/962b27c3fe114ae9b7e8c0ff47035da9 2024-11-21T21:25:21,505 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/A of d6ccac84b1b168c8055162a63f397098 into 962b27c3fe114ae9b7e8c0ff47035da9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:21,505 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:21,505 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/A, priority=12, startTime=1732224321466; duration=0sec 2024-11-21T21:25:21,505 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:21,505 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:A 2024-11-21T21:25:21,505 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:25:21,506 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/B of d6ccac84b1b168c8055162a63f397098 into 6b510b2f55644b5bb8d591596b00d80f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:21,506 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:21,506 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/B, priority=12, startTime=1732224321466; duration=0sec 2024-11-21T21:25:21,506 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:25:21,506 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): d6ccac84b1b168c8055162a63f397098/C is initiating minor compaction (all files) 2024-11-21T21:25:21,506 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:21,506 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ccac84b1b168c8055162a63f397098/C in TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:21,506 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:B 2024-11-21T21:25:21,506 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/4b2c6a0446e845dfa01206eaeaf456dd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bcface94caab466cb9e747d4482493d0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/91c00dea314e4962bbc65b7e4e5a1428, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9098d42cbeeb4f759c3b29830a5e2d8c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp, totalSize=48.9 K 2024-11-21T21:25:21,507 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2c6a0446e845dfa01206eaeaf456dd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732224317327 2024-11-21T21:25:21,507 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcface94caab466cb9e747d4482493d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732224317459 2024-11-21T21:25:21,507 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c00dea314e4962bbc65b7e4e5a1428, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732224318112 2024-11-21T21:25:21,507 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9098d42cbeeb4f759c3b29830a5e2d8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732224320294 2024-11-21T21:25:21,515 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ccac84b1b168c8055162a63f397098#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:21,516 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/46d0b244e2c64ddea3094e685f104430 is 50, key is test_row_0/C:col10/1732224320930/Put/seqid=0 2024-11-21T21:25:21,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742181_1357 (size=13323) 2024-11-21T21:25:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:21,602 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:21,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T21:25:21,602 INFO [Thread-1218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1218. 2024-11-21T21:25:21,602 INFO [Thread-1224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1224. 2024-11-21T21:25:21,602 DEBUG [Thread-1218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,602 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,602 DEBUG [Thread-1224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,602 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,602 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,602 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,602 INFO [Thread-1220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1220. 2024-11-21T21:25:21,602 DEBUG [Thread-1220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:21,602 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,602 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:21,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:21,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3d1a12467ea84cdb8c8716c1edaa9be5 is 50, key is test_row_0/A:col10/1732224320947/Put/seqid=0 2024-11-21T21:25:21,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742182_1358 (size=12301) 2024-11-21T21:25:21,924 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/46d0b244e2c64ddea3094e685f104430 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/46d0b244e2c64ddea3094e685f104430 2024-11-21T21:25:21,929 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ccac84b1b168c8055162a63f397098/C of d6ccac84b1b168c8055162a63f397098 into 46d0b244e2c64ddea3094e685f104430(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:21,930 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:21,930 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., storeName=d6ccac84b1b168c8055162a63f397098/C, priority=12, startTime=1732224321466; duration=0sec 2024-11-21T21:25:21,930 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:21,930 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ccac84b1b168c8055162a63f397098:C 2024-11-21T21:25:22,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3d1a12467ea84cdb8c8716c1edaa9be5 2024-11-21T21:25:22,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/66787395bc534d5e8c69acb3306b98ad is 50, key is test_row_0/B:col10/1732224320947/Put/seqid=0 2024-11-21T21:25:22,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742183_1359 (size=12301) 2024-11-21T21:25:22,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/66787395bc534d5e8c69acb3306b98ad 2024-11-21T21:25:22,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/083419876c1845fa88309e2a9bdd7696 is 50, key is test_row_0/C:col10/1732224320947/Put/seqid=0 2024-11-21T21:25:22,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742184_1360 (size=12301) 2024-11-21T21:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:25:22,370 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:22,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/083419876c1845fa88309e2a9bdd7696 2024-11-21T21:25:22,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/3d1a12467ea84cdb8c8716c1edaa9be5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3d1a12467ea84cdb8c8716c1edaa9be5 2024-11-21T21:25:22,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3d1a12467ea84cdb8c8716c1edaa9be5, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T21:25:22,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/66787395bc534d5e8c69acb3306b98ad as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/66787395bc534d5e8c69acb3306b98ad 2024-11-21T21:25:22,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/66787395bc534d5e8c69acb3306b98ad, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T21:25:22,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/083419876c1845fa88309e2a9bdd7696 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/083419876c1845fa88309e2a9bdd7696 2024-11-21T21:25:22,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/083419876c1845fa88309e2a9bdd7696, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T21:25:22,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for d6ccac84b1b168c8055162a63f397098 in 883ms, sequenceid=457, compaction requested=false 2024-11-21T21:25:22,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ccac84b1b168c8055162a63f397098: 2024-11-21T21:25:24,152 INFO [Thread-1222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1222. 2024-11-21T21:25:24,152 DEBUG [Thread-1222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:24,152 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:24,152 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:24,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:24,161 INFO [Thread-1226 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1226. 2024-11-21T21:25:24,161 DEBUG [Thread-1226 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:24,161 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:24,161 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:24,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4366 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4196 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4066 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4382 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4192 2024-11-21T21:25:24,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:25:24,162 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:25:24,162 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:25:24,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:24,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:24,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:24,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:25:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:25:24,167 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224324166"}]},"ts":"1732224324166"} 2024-11-21T21:25:24,168 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:25:24,168 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:25:24,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:25:24,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, UNASSIGN}] 2024-11-21T21:25:24,171 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, UNASSIGN 2024-11-21T21:25:24,172 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=d6ccac84b1b168c8055162a63f397098, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:24,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, UNASSIGN because future has completed 2024-11-21T21:25:24,173 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:25:24,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:25:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:25:24,326 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:24,326 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:25:24,326 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing d6ccac84b1b168c8055162a63f397098, disabling compactions & flushes 2024-11-21T21:25:24,326 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:24,326 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:24,326 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. after waiting 0 ms 2024-11-21T21:25:24,326 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:24,326 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing d6ccac84b1b168c8055162a63f397098 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-21T21:25:24,333 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/32bf915df6c945b2b237bd6ba7c777a9 is 50, key is test_row_0/A:col10/1732224324160/Put/seqid=0 2024-11-21T21:25:24,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742185_1361 (size=7415) 2024-11-21T21:25:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:25:24,737 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/32bf915df6c945b2b237bd6ba7c777a9 2024-11-21T21:25:24,746 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/b5fd9f280ea8466faa1fdf3aa86d58e7 is 50, key is test_row_0/B:col10/1732224324160/Put/seqid=0 2024-11-21T21:25:24,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742186_1362 (size=7415) 2024-11-21T21:25:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:25:25,152 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/b5fd9f280ea8466faa1fdf3aa86d58e7 2024-11-21T21:25:25,163 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bbedd948cda34ff3914db064a630e415 is 50, key is test_row_0/C:col10/1732224324160/Put/seqid=0 2024-11-21T21:25:25,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742187_1363 (size=7415) 2024-11-21T21:25:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:25:25,568 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bbedd948cda34ff3914db064a630e415 2024-11-21T21:25:25,574 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/A/32bf915df6c945b2b237bd6ba7c777a9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/32bf915df6c945b2b237bd6ba7c777a9 2024-11-21T21:25:25,579 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/32bf915df6c945b2b237bd6ba7c777a9, entries=50, sequenceid=463, filesize=7.2 K 2024-11-21T21:25:25,580 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/B/b5fd9f280ea8466faa1fdf3aa86d58e7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/b5fd9f280ea8466faa1fdf3aa86d58e7 2024-11-21T21:25:25,584 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/b5fd9f280ea8466faa1fdf3aa86d58e7, entries=50, sequenceid=463, filesize=7.2 K 2024-11-21T21:25:25,585 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/.tmp/C/bbedd948cda34ff3914db064a630e415 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bbedd948cda34ff3914db064a630e415 2024-11-21T21:25:25,588 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bbedd948cda34ff3914db064a630e415, entries=50, sequenceid=463, filesize=7.2 K 2024-11-21T21:25:25,589 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for d6ccac84b1b168c8055162a63f397098 in 1263ms, sequenceid=463, compaction requested=true 2024-11-21T21:25:25,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3bc21805f4034d35a13b8d8fcb29e90f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/fdd8d6474f954db98954356f96713a93, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/6bd9ebdaccff41a7be217e7dcea3b79f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/ae9699ffc53f4734b152c6d267880b2d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d3699e122914cfca38a1d3799cdd49f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/c0fef7e53c1f4b5fa206bb217476b002, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/541e79c11aa240b48112f26379d76812, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/796d441b4f434ff09956d01561dd597d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bf4a883379d947b08099faf0d12d627b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/60b3fdfb0ff240c4a0292985eb47cd27, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d8ec24ea209a46d7bb68e3f855f7505b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2e70157496db4b5583fa3c20751c8a1e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/aa24fc2d1576483fb276909d7b4af30f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e3fb8f2b352b45ff99e6b270d98efaf6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bd26926b053040258de7f395b5de52c6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d68f40d2f120424a9f77c3bd2b0aff53, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0a463e0d09794dfc863ab76e165078e0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/7e3c18c25c3a4a29b69615c7401a6ffc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3e19052fdff0423bbaf74be30f2d4c35, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e381f41ce3104cd1b1d2879b599bb6ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/082a58cc04c74a18a6199477d73e7084, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2bfbc404d9e4424f871b0094e001732f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0e966588e8354e1ebcd30265c0caccfa, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/a150f62d3c6945c584583641b0cd58fe, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2960bf2a2466438191ee74ed8a4415f2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/728a52d617ad4af2a662ed91389382b8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/cece9251367f46fe8d191421f257d273, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e9465a0f711a4d62b1c128b8f8f3e5a5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/5ac777b72ac34fd2bc328af89d1f71d1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d55259659914708ae0995af3c158ce2] to archive 2024-11-21T21:25:25,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:25:25,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3bc21805f4034d35a13b8d8fcb29e90f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3bc21805f4034d35a13b8d8fcb29e90f 2024-11-21T21:25:25,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/fdd8d6474f954db98954356f96713a93 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/fdd8d6474f954db98954356f96713a93 2024-11-21T21:25:25,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/6bd9ebdaccff41a7be217e7dcea3b79f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/6bd9ebdaccff41a7be217e7dcea3b79f 2024-11-21T21:25:25,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/ae9699ffc53f4734b152c6d267880b2d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/ae9699ffc53f4734b152c6d267880b2d 2024-11-21T21:25:25,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d3699e122914cfca38a1d3799cdd49f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d3699e122914cfca38a1d3799cdd49f 2024-11-21T21:25:25,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/c0fef7e53c1f4b5fa206bb217476b002 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/c0fef7e53c1f4b5fa206bb217476b002 2024-11-21T21:25:25,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/541e79c11aa240b48112f26379d76812 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/541e79c11aa240b48112f26379d76812 2024-11-21T21:25:25,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/796d441b4f434ff09956d01561dd597d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/796d441b4f434ff09956d01561dd597d 2024-11-21T21:25:25,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bf4a883379d947b08099faf0d12d627b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bf4a883379d947b08099faf0d12d627b 2024-11-21T21:25:25,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/60b3fdfb0ff240c4a0292985eb47cd27 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/60b3fdfb0ff240c4a0292985eb47cd27 2024-11-21T21:25:25,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d8ec24ea209a46d7bb68e3f855f7505b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d8ec24ea209a46d7bb68e3f855f7505b 2024-11-21T21:25:25,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2e70157496db4b5583fa3c20751c8a1e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2e70157496db4b5583fa3c20751c8a1e 2024-11-21T21:25:25,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/aa24fc2d1576483fb276909d7b4af30f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/aa24fc2d1576483fb276909d7b4af30f 2024-11-21T21:25:25,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e3fb8f2b352b45ff99e6b270d98efaf6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e3fb8f2b352b45ff99e6b270d98efaf6 2024-11-21T21:25:25,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bd26926b053040258de7f395b5de52c6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/bd26926b053040258de7f395b5de52c6 2024-11-21T21:25:25,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d68f40d2f120424a9f77c3bd2b0aff53 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/d68f40d2f120424a9f77c3bd2b0aff53 2024-11-21T21:25:25,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0a463e0d09794dfc863ab76e165078e0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0a463e0d09794dfc863ab76e165078e0 2024-11-21T21:25:25,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/7e3c18c25c3a4a29b69615c7401a6ffc to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/7e3c18c25c3a4a29b69615c7401a6ffc 2024-11-21T21:25:25,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3e19052fdff0423bbaf74be30f2d4c35 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3e19052fdff0423bbaf74be30f2d4c35 2024-11-21T21:25:25,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e381f41ce3104cd1b1d2879b599bb6ff to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e381f41ce3104cd1b1d2879b599bb6ff 2024-11-21T21:25:25,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/082a58cc04c74a18a6199477d73e7084 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/082a58cc04c74a18a6199477d73e7084 2024-11-21T21:25:25,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2bfbc404d9e4424f871b0094e001732f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2bfbc404d9e4424f871b0094e001732f 2024-11-21T21:25:25,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0e966588e8354e1ebcd30265c0caccfa to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0e966588e8354e1ebcd30265c0caccfa 2024-11-21T21:25:25,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/a150f62d3c6945c584583641b0cd58fe to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/a150f62d3c6945c584583641b0cd58fe 2024-11-21T21:25:25,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2960bf2a2466438191ee74ed8a4415f2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/2960bf2a2466438191ee74ed8a4415f2 2024-11-21T21:25:25,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/728a52d617ad4af2a662ed91389382b8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/728a52d617ad4af2a662ed91389382b8 2024-11-21T21:25:25,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/cece9251367f46fe8d191421f257d273 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/cece9251367f46fe8d191421f257d273 2024-11-21T21:25:25,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e9465a0f711a4d62b1c128b8f8f3e5a5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/e9465a0f711a4d62b1c128b8f8f3e5a5 2024-11-21T21:25:25,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/5ac777b72ac34fd2bc328af89d1f71d1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/5ac777b72ac34fd2bc328af89d1f71d1 2024-11-21T21:25:25,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d55259659914708ae0995af3c158ce2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/0d55259659914708ae0995af3c158ce2 2024-11-21T21:25:25,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/40a1ca7988a64b9285f8796ae847d74d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3b5e0460e0214b07a6d4561c9106237b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/92d81f799f104638b04aa928bfbaf17c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/53dd08fc2de44bd6bb1988d1631fbdee, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/0e8e6b6caeae436eb7f0f62189f1e8f2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/72d3f9e7d1ee4280949b85de9447503e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/fb25b3aa80f44de29f9896c9b6ba9e46, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/06541a3aa91845cfb0d156974677a18b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1fa1aee16d6c429ca09171ca8aeeb425, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4e2718261f724fa7bfa4d7902ae96a85, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/8945b366dccc40208ca5a1189a10e5e3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4a32e7c21e0f4627abd82947e0d69158, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1bdeab71875c4933922390916abcb06a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/934cc046ea394b5cae951c3358c0b53f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/e73dc7b552c647368e4cbbdd492bd5f8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/776537fe4d5347379375538d980e47f6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/c7de4ab5701e4bbca0839426b273ac54, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/a1b4c587d6fe40ff9ce2106ccf676e2e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/dbd0c910da954cf998c003cd80d10321, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/15e8ddbb07024e938b8dc5d3f96c2e15, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/af84e6f4065b4f98b9a0bed69ef1b63b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/31da16f6b7ad46489c72a57d0cee79c1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/09d3b4e9769146ff9930f42f002a6664, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3fee5465eb5d40f89fe282ce9ea0ceef, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/2c5327a07b144ae2afaf74695aebcb37, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/cca6985fd3fc4e2585ced500e6accdb8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/ea36ce5664ff45c080a956061697fd55, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/7c8b7ce6ab704dac8150ffb2fcbacabf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/79e1e007fd6f4d4690ca953f8700b8ca, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3c935a1731d34058a4d5e44286d22dae] to archive 2024-11-21T21:25:25,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:25:25,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/40a1ca7988a64b9285f8796ae847d74d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/40a1ca7988a64b9285f8796ae847d74d 2024-11-21T21:25:25,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3b5e0460e0214b07a6d4561c9106237b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3b5e0460e0214b07a6d4561c9106237b 2024-11-21T21:25:25,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/92d81f799f104638b04aa928bfbaf17c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/92d81f799f104638b04aa928bfbaf17c 2024-11-21T21:25:25,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/53dd08fc2de44bd6bb1988d1631fbdee to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/53dd08fc2de44bd6bb1988d1631fbdee 2024-11-21T21:25:25,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/0e8e6b6caeae436eb7f0f62189f1e8f2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/0e8e6b6caeae436eb7f0f62189f1e8f2 2024-11-21T21:25:25,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/72d3f9e7d1ee4280949b85de9447503e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/72d3f9e7d1ee4280949b85de9447503e 2024-11-21T21:25:25,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/fb25b3aa80f44de29f9896c9b6ba9e46 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/fb25b3aa80f44de29f9896c9b6ba9e46 2024-11-21T21:25:25,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/06541a3aa91845cfb0d156974677a18b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/06541a3aa91845cfb0d156974677a18b 2024-11-21T21:25:25,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1fa1aee16d6c429ca09171ca8aeeb425 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1fa1aee16d6c429ca09171ca8aeeb425 2024-11-21T21:25:25,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4e2718261f724fa7bfa4d7902ae96a85 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4e2718261f724fa7bfa4d7902ae96a85 2024-11-21T21:25:25,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/8945b366dccc40208ca5a1189a10e5e3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/8945b366dccc40208ca5a1189a10e5e3 2024-11-21T21:25:25,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4a32e7c21e0f4627abd82947e0d69158 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/4a32e7c21e0f4627abd82947e0d69158 2024-11-21T21:25:25,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1bdeab71875c4933922390916abcb06a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/1bdeab71875c4933922390916abcb06a 2024-11-21T21:25:25,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/934cc046ea394b5cae951c3358c0b53f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/934cc046ea394b5cae951c3358c0b53f 2024-11-21T21:25:25,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/e73dc7b552c647368e4cbbdd492bd5f8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/e73dc7b552c647368e4cbbdd492bd5f8 2024-11-21T21:25:25,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/776537fe4d5347379375538d980e47f6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/776537fe4d5347379375538d980e47f6 2024-11-21T21:25:25,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/c7de4ab5701e4bbca0839426b273ac54 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/c7de4ab5701e4bbca0839426b273ac54 2024-11-21T21:25:25,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/a1b4c587d6fe40ff9ce2106ccf676e2e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/a1b4c587d6fe40ff9ce2106ccf676e2e 2024-11-21T21:25:25,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/dbd0c910da954cf998c003cd80d10321 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/dbd0c910da954cf998c003cd80d10321 2024-11-21T21:25:25,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/15e8ddbb07024e938b8dc5d3f96c2e15 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/15e8ddbb07024e938b8dc5d3f96c2e15 2024-11-21T21:25:25,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/af84e6f4065b4f98b9a0bed69ef1b63b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/af84e6f4065b4f98b9a0bed69ef1b63b 2024-11-21T21:25:25,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/31da16f6b7ad46489c72a57d0cee79c1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/31da16f6b7ad46489c72a57d0cee79c1 2024-11-21T21:25:25,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/09d3b4e9769146ff9930f42f002a6664 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/09d3b4e9769146ff9930f42f002a6664 2024-11-21T21:25:25,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3fee5465eb5d40f89fe282ce9ea0ceef to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3fee5465eb5d40f89fe282ce9ea0ceef 2024-11-21T21:25:25,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/2c5327a07b144ae2afaf74695aebcb37 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/2c5327a07b144ae2afaf74695aebcb37 2024-11-21T21:25:25,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/cca6985fd3fc4e2585ced500e6accdb8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/cca6985fd3fc4e2585ced500e6accdb8 2024-11-21T21:25:25,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/ea36ce5664ff45c080a956061697fd55 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/ea36ce5664ff45c080a956061697fd55 2024-11-21T21:25:25,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/7c8b7ce6ab704dac8150ffb2fcbacabf to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/7c8b7ce6ab704dac8150ffb2fcbacabf 2024-11-21T21:25:25,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/79e1e007fd6f4d4690ca953f8700b8ca to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/79e1e007fd6f4d4690ca953f8700b8ca 2024-11-21T21:25:25,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3c935a1731d34058a4d5e44286d22dae to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/3c935a1731d34058a4d5e44286d22dae 2024-11-21T21:25:25,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ec5a3500777b4f488cd9560f6c6b3120, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/fd5cb062ac794e418d5554324663ac16, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/8a8e17f75d5c4c02a973c897992b0105, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/389d7cd39d4249339538d5325e9adb3d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/5be85ae02dea466b8c965f033569a69b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ee6809bf50bf4ca3a6094e3fdaee41a7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/40ac7b43f67a454cb6032b0f682d711a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7c581dc42a654542a73562a15a4e20a7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/49b3e9b9a0c348ff9b959ccea9e663fd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/1f9730ad4f834fb4bfe6037c8fd178a4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/84e5ff25421c40efb68c8f7a3c13fd53, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7752ad8d920b4dab93452eedb6a2f371, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ad839f1b925f41969b115d2d31c75fe2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/38621f9167b64b69b861c92d07198c3a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ae4e81af195c45dba29b337eb3abfe37, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9cf370b9f206499d9131e849d72d6ea5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/e7129e2042314905872077b1a82fad94, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/da4c0eb3f3a6453a92389e33a4cbf0dd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/65b3dbeb16ce4d2aa0852e7642c7f0cb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/d3d6276c41a44aa08c24e1a2731ae09a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/6880078e31284fa5bb32510a7f7c9d59, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/962d618e03e045268bdeba2fd86e09ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/28150471c62143ba8f4c539b67f20d6d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/153af8f0d87447d28249ff0ab63fce94, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bae969c73bed4462a7cc5f33c4a7f643, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/4b2c6a0446e845dfa01206eaeaf456dd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/186814e053ab46eb8936552a7eb28f7a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bcface94caab466cb9e747d4482493d0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/91c00dea314e4962bbc65b7e4e5a1428, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9098d42cbeeb4f759c3b29830a5e2d8c] to archive 2024-11-21T21:25:25,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:25:25,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ec5a3500777b4f488cd9560f6c6b3120 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ec5a3500777b4f488cd9560f6c6b3120 2024-11-21T21:25:25,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/fd5cb062ac794e418d5554324663ac16 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/fd5cb062ac794e418d5554324663ac16 2024-11-21T21:25:25,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/8a8e17f75d5c4c02a973c897992b0105 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/8a8e17f75d5c4c02a973c897992b0105 2024-11-21T21:25:25,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/389d7cd39d4249339538d5325e9adb3d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/389d7cd39d4249339538d5325e9adb3d 2024-11-21T21:25:25,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/5be85ae02dea466b8c965f033569a69b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/5be85ae02dea466b8c965f033569a69b 2024-11-21T21:25:25,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ee6809bf50bf4ca3a6094e3fdaee41a7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ee6809bf50bf4ca3a6094e3fdaee41a7 2024-11-21T21:25:25,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/40ac7b43f67a454cb6032b0f682d711a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/40ac7b43f67a454cb6032b0f682d711a 2024-11-21T21:25:25,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7c581dc42a654542a73562a15a4e20a7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7c581dc42a654542a73562a15a4e20a7 2024-11-21T21:25:25,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/49b3e9b9a0c348ff9b959ccea9e663fd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/49b3e9b9a0c348ff9b959ccea9e663fd 2024-11-21T21:25:25,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/1f9730ad4f834fb4bfe6037c8fd178a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/1f9730ad4f834fb4bfe6037c8fd178a4 2024-11-21T21:25:25,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/84e5ff25421c40efb68c8f7a3c13fd53 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/84e5ff25421c40efb68c8f7a3c13fd53 2024-11-21T21:25:25,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7752ad8d920b4dab93452eedb6a2f371 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/7752ad8d920b4dab93452eedb6a2f371 2024-11-21T21:25:25,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ad839f1b925f41969b115d2d31c75fe2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ad839f1b925f41969b115d2d31c75fe2 2024-11-21T21:25:25,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/38621f9167b64b69b861c92d07198c3a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/38621f9167b64b69b861c92d07198c3a 2024-11-21T21:25:25,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ae4e81af195c45dba29b337eb3abfe37 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/ae4e81af195c45dba29b337eb3abfe37 2024-11-21T21:25:25,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9cf370b9f206499d9131e849d72d6ea5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9cf370b9f206499d9131e849d72d6ea5 2024-11-21T21:25:25,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/e7129e2042314905872077b1a82fad94 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/e7129e2042314905872077b1a82fad94 2024-11-21T21:25:25,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/da4c0eb3f3a6453a92389e33a4cbf0dd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/da4c0eb3f3a6453a92389e33a4cbf0dd 2024-11-21T21:25:25,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/65b3dbeb16ce4d2aa0852e7642c7f0cb to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/65b3dbeb16ce4d2aa0852e7642c7f0cb 2024-11-21T21:25:25,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/d3d6276c41a44aa08c24e1a2731ae09a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/d3d6276c41a44aa08c24e1a2731ae09a 2024-11-21T21:25:25,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/6880078e31284fa5bb32510a7f7c9d59 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/6880078e31284fa5bb32510a7f7c9d59 2024-11-21T21:25:25,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/962d618e03e045268bdeba2fd86e09ff to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/962d618e03e045268bdeba2fd86e09ff 2024-11-21T21:25:25,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/28150471c62143ba8f4c539b67f20d6d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/28150471c62143ba8f4c539b67f20d6d 2024-11-21T21:25:25,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/153af8f0d87447d28249ff0ab63fce94 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/153af8f0d87447d28249ff0ab63fce94 2024-11-21T21:25:25,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bae969c73bed4462a7cc5f33c4a7f643 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bae969c73bed4462a7cc5f33c4a7f643 2024-11-21T21:25:25,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/4b2c6a0446e845dfa01206eaeaf456dd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/4b2c6a0446e845dfa01206eaeaf456dd 2024-11-21T21:25:25,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/186814e053ab46eb8936552a7eb28f7a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/186814e053ab46eb8936552a7eb28f7a 2024-11-21T21:25:25,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bcface94caab466cb9e747d4482493d0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bcface94caab466cb9e747d4482493d0 2024-11-21T21:25:25,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/91c00dea314e4962bbc65b7e4e5a1428 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/91c00dea314e4962bbc65b7e4e5a1428 2024-11-21T21:25:25,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9098d42cbeeb4f759c3b29830a5e2d8c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/9098d42cbeeb4f759c3b29830a5e2d8c 2024-11-21T21:25:25,674 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/recovered.edits/466.seqid, newMaxSeqId=466, maxSeqId=1 2024-11-21T21:25:25,674 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. 2024-11-21T21:25:25,674 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for d6ccac84b1b168c8055162a63f397098: Waiting for close lock at 1732224324326Running coprocessor pre-close hooks at 1732224324326Disabling compacts and flushes for region at 1732224324326Disabling writes for close at 1732224324326Obtaining lock to block concurrent updates at 1732224324326Preparing flush snapshotting stores in d6ccac84b1b168c8055162a63f397098 at 1732224324326Finished memstore snapshotting TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732224324327 (+1 ms)Flushing stores of TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098. at 1732224324327Flushing d6ccac84b1b168c8055162a63f397098/A: creating writer at 1732224324327Flushing d6ccac84b1b168c8055162a63f397098/A: appending metadata at 1732224324331 (+4 ms)Flushing d6ccac84b1b168c8055162a63f397098/A: closing flushed file at 1732224324331Flushing d6ccac84b1b168c8055162a63f397098/B: creating writer at 1732224324742 (+411 ms)Flushing d6ccac84b1b168c8055162a63f397098/B: appending metadata at 1732224324745 (+3 ms)Flushing d6ccac84b1b168c8055162a63f397098/B: closing flushed file at 1732224324745Flushing d6ccac84b1b168c8055162a63f397098/C: creating writer at 1732224325158 (+413 ms)Flushing d6ccac84b1b168c8055162a63f397098/C: appending metadata at 1732224325162 (+4 ms)Flushing d6ccac84b1b168c8055162a63f397098/C: closing flushed file at 1732224325162Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76c63d37: reopening flushed file at 1732224325573 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c87e03e: reopening flushed file at 1732224325580 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b87300: reopening flushed file at 1732224325584 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for d6ccac84b1b168c8055162a63f397098 in 1263ms, sequenceid=463, compaction requested=true at 1732224325589 (+5 ms)Writing region close event to WAL at 1732224325671 (+82 ms)Running coprocessor post-close hooks at 1732224325674 (+3 ms)Closed at 1732224325674 2024-11-21T21:25:25,676 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:25,676 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=d6ccac84b1b168c8055162a63f397098, regionState=CLOSED 2024-11-21T21:25:25,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:25:25,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T21:25:25,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure d6ccac84b1b168c8055162a63f397098, server=849dee24a242,34123,1732224229084 in 1.5050 sec 2024-11-21T21:25:25,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-21T21:25:25,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ccac84b1b168c8055162a63f397098, UNASSIGN in 1.5100 sec 2024-11-21T21:25:25,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T21:25:25,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5130 sec 2024-11-21T21:25:25,685 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224325685"}]},"ts":"1732224325685"} 2024-11-21T21:25:25,686 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:25:25,687 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:25:25,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-21T21:25:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:25:26,310 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:26,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:25:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:26,312 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T21:25:26,313 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:26,315 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:26,317 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/recovered.edits] 2024-11-21T21:25:26,319 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/32bf915df6c945b2b237bd6ba7c777a9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/32bf915df6c945b2b237bd6ba7c777a9 2024-11-21T21:25:26,320 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3d1a12467ea84cdb8c8716c1edaa9be5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/3d1a12467ea84cdb8c8716c1edaa9be5 2024-11-21T21:25:26,321 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/962b27c3fe114ae9b7e8c0ff47035da9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/A/962b27c3fe114ae9b7e8c0ff47035da9 2024-11-21T21:25:26,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/66787395bc534d5e8c69acb3306b98ad to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/66787395bc534d5e8c69acb3306b98ad 2024-11-21T21:25:26,324 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/6b510b2f55644b5bb8d591596b00d80f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/6b510b2f55644b5bb8d591596b00d80f 2024-11-21T21:25:26,325 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/b5fd9f280ea8466faa1fdf3aa86d58e7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/B/b5fd9f280ea8466faa1fdf3aa86d58e7 2024-11-21T21:25:26,327 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/083419876c1845fa88309e2a9bdd7696 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/083419876c1845fa88309e2a9bdd7696 2024-11-21T21:25:26,328 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/46d0b244e2c64ddea3094e685f104430 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/46d0b244e2c64ddea3094e685f104430 2024-11-21T21:25:26,329 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bbedd948cda34ff3914db064a630e415 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/C/bbedd948cda34ff3914db064a630e415 2024-11-21T21:25:26,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/recovered.edits/466.seqid to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098/recovered.edits/466.seqid 2024-11-21T21:25:26,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/d6ccac84b1b168c8055162a63f397098 2024-11-21T21:25:26,332 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:25:26,335 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:26,337 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:25:26,339 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:25:26,340 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:26,340 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:25:26,340 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732224326340"}]},"ts":"9223372036854775807"} 2024-11-21T21:25:26,342 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:25:26,342 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d6ccac84b1b168c8055162a63f397098, NAME => 'TestAcidGuarantees,,1732224298788.d6ccac84b1b168c8055162a63f397098.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:25:26,342 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:25:26,343 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732224326342"}]},"ts":"9223372036854775807"} 2024-11-21T21:25:26,345 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:25:26,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:26,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-21T21:25:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T21:25:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:25:26,424 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:26,432 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1390536951_22 at /127.0.0.1:39352 [Waiting for operation #337] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-185090283_22 at /127.0.0.1:33686 [Waiting for operation #270] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=12585 (was 12294) - AvailableMemoryMB LEAK? - 2024-11-21T21:25:26,443 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=12581 2024-11-21T21:25:26,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:25:26,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:25:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:26,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:25:26,448 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:26,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-21T21:25:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-21T21:25:26,449 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:25:26,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742188_1364 (size=959) 2024-11-21T21:25:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-21T21:25:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-21T21:25:26,859 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:25:26,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742189_1365 (size=53) 2024-11-21T21:25:26,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:25:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 58ad0d609f8c768bc6016e57e5d831ac, disabling compactions & flushes 2024-11-21T21:25:26,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. after waiting 0 ms 2024-11-21T21:25:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:26,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 58ad0d609f8c768bc6016e57e5d831ac: Waiting for close lock at 1732224326869Disabling compacts and flushes for region at 1732224326869Disabling writes for close at 1732224326869Writing region close event to WAL at 1732224326869Closed at 1732224326869 2024-11-21T21:25:26,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:25:26,870 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732224326870"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732224326870"}]},"ts":"1732224326870"} 2024-11-21T21:25:26,871 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:25:26,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:25:26,873 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224326872"}]},"ts":"1732224326872"} 2024-11-21T21:25:26,874 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:25:26,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, ASSIGN}] 2024-11-21T21:25:26,876 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, ASSIGN 2024-11-21T21:25:26,877 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, ASSIGN; state=OFFLINE, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=false 2024-11-21T21:25:27,027 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:27,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, ASSIGN because future has completed 2024-11-21T21:25:27,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:25:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-21T21:25:27,185 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:27,185 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:25:27,185 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,185 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:25:27,186 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,186 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,187 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,188 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58ad0d609f8c768bc6016e57e5d831ac columnFamilyName A 2024-11-21T21:25:27,188 DEBUG [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:27,189 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(327): Store=58ad0d609f8c768bc6016e57e5d831ac/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:27,189 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,190 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58ad0d609f8c768bc6016e57e5d831ac columnFamilyName B 2024-11-21T21:25:27,190 DEBUG [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:27,190 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(327): Store=58ad0d609f8c768bc6016e57e5d831ac/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:27,190 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,191 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58ad0d609f8c768bc6016e57e5d831ac columnFamilyName C 2024-11-21T21:25:27,191 DEBUG [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:27,192 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(327): Store=58ad0d609f8c768bc6016e57e5d831ac/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:27,192 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,193 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,193 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,194 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,194 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,194 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:25:27,196 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,198 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:25:27,198 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 58ad0d609f8c768bc6016e57e5d831ac; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70045307, jitterRate=0.04375641047954559}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:25:27,198 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:27,199 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 58ad0d609f8c768bc6016e57e5d831ac: Running coprocessor pre-open hook at 1732224327186Writing region info on filesystem at 1732224327186Initializing all the Stores at 1732224327186Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224327186Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224327187 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224327187Cleaning up temporary data from old regions at 1732224327194 (+7 ms)Running coprocessor post-open hooks at 1732224327198 (+4 ms)Region opened successfully at 1732224327199 (+1 ms) 2024-11-21T21:25:27,200 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., pid=87, masterSystemTime=1732224327182 2024-11-21T21:25:27,202 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:27,202 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:27,203 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:27,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:25:27,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T21:25:27,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 in 176 msec 2024-11-21T21:25:27,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-21T21:25:27,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, ASSIGN in 333 msec 2024-11-21T21:25:27,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:25:27,211 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224327211"}]},"ts":"1732224327211"} 2024-11-21T21:25:27,213 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:25:27,214 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:25:27,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 769 msec 2024-11-21T21:25:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-21T21:25:27,590 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:27,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:27,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:27,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:27,592 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511f59b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:27,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bd8e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:27,595 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:27,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:27,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:25:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,35047,1732224227985 2024-11-21T21:25:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7dbb7d48 2024-11-21T21:25:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:25:27,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:25:27,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:25:27,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:25:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:27,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742190_1366 (size=995) 2024-11-21T21:25:28,012 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T21:25:28,012 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T21:25:28,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:25:28,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, REOPEN/MOVE}] 2024-11-21T21:25:28,020 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, REOPEN/MOVE 2024-11-21T21:25:28,021 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, REOPEN/MOVE because future has completed 2024-11-21T21:25:28,023 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:25:28,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:25:28,176 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,176 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:25:28,176 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 58ad0d609f8c768bc6016e57e5d831ac, disabling compactions & flushes 2024-11-21T21:25:28,176 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,176 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,176 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. after waiting 0 ms 2024-11-21T21:25:28,176 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,179 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T21:25:28,180 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,180 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 58ad0d609f8c768bc6016e57e5d831ac: Waiting for close lock at 1732224328176Running coprocessor pre-close hooks at 1732224328176Disabling compacts and flushes for region at 1732224328176Disabling writes for close at 1732224328176Writing region close event to WAL at 1732224328177 (+1 ms)Running coprocessor post-close hooks at 1732224328180 (+3 ms)Closed at 1732224328180 2024-11-21T21:25:28,180 WARN [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 58ad0d609f8c768bc6016e57e5d831ac to self. 2024-11-21T21:25:28,181 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,182 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=CLOSED 2024-11-21T21:25:28,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:25:28,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-21T21:25:28,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 in 161 msec 2024-11-21T21:25:28,185 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, REOPEN/MOVE; state=CLOSED, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=true 2024-11-21T21:25:28,336 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, REOPEN/MOVE because future has completed 2024-11-21T21:25:28,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:25:28,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:25:28,492 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,492 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:25:28,492 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,492 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:25:28,492 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,492 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,493 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,494 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58ad0d609f8c768bc6016e57e5d831ac columnFamilyName A 2024-11-21T21:25:28,494 DEBUG [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:28,495 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(327): Store=58ad0d609f8c768bc6016e57e5d831ac/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:28,495 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,496 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58ad0d609f8c768bc6016e57e5d831ac columnFamilyName B 2024-11-21T21:25:28,496 DEBUG [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:28,496 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(327): Store=58ad0d609f8c768bc6016e57e5d831ac/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:28,496 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,497 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58ad0d609f8c768bc6016e57e5d831ac columnFamilyName C 2024-11-21T21:25:28,497 DEBUG [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:28,497 INFO [StoreOpener-58ad0d609f8c768bc6016e57e5d831ac-1 {}] regionserver.HStore(327): Store=58ad0d609f8c768bc6016e57e5d831ac/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:28,497 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,498 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,498 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,499 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,499 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,499 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:25:28,500 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,501 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 58ad0d609f8c768bc6016e57e5d831ac; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65663168, jitterRate=-0.02154254913330078}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:25:28,501 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,501 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 58ad0d609f8c768bc6016e57e5d831ac: Running coprocessor pre-open hook at 1732224328492Writing region info on filesystem at 1732224328492Initializing all the Stores at 1732224328493 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224328493Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224328493Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224328493Cleaning up temporary data from old regions at 1732224328499 (+6 ms)Running coprocessor post-open hooks at 1732224328501 (+2 ms)Region opened successfully at 1732224328501 2024-11-21T21:25:28,502 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., pid=92, masterSystemTime=1732224328489 2024-11-21T21:25:28,503 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,503 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,503 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=OPEN, openSeqNum=5, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:25:28,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-21T21:25:28,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 in 167 msec 2024-11-21T21:25:28,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-21T21:25:28,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, REOPEN/MOVE in 488 msec 2024-11-21T21:25:28,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-21T21:25:28,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-21T21:25:28,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 910 msec 2024-11-21T21:25:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T21:25:28,513 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:28,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,514 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2af295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23454df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,517 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e3d620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2056feed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad0bad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,522 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215b6bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,523 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ad45985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55eb7700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,527 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfda626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,529 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65fa6d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,532 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af95d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1908f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,536 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b05e24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,538 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d070ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,540 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@207c9e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,542 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2cbb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75840ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b43d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:28,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:28,561 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a282b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:28,563 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de03d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:28,564 DEBUG [Thread-1665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,565 DEBUG [Thread-1671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,576 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,578 DEBUG [Thread-1656 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,578 DEBUG [Thread-1656 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,589 DEBUG [Thread-1673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,591 DEBUG [Thread-1669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,596 DEBUG [Thread-1667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,602 DEBUG [Thread-1673 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,602 DEBUG [Thread-1673 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:28,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:25:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,616 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,616 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,617 DEBUG [Thread-1673 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,624 DEBUG [Thread-1660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,626 DEBUG [Thread-1660 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,640 DEBUG [Thread-1662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:28,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:25:28,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-21T21:25:28,660 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:25:28,661 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:28,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:28,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d6dcbc7297c54b9d921b29ce11d8077f_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224328606/Put/seqid=0 2024-11-21T21:25:28,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742191_1367 (size=12154) 2024-11-21T21:25:28,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224388693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224388696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224388698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224388700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224388701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,716 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:28,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d6dcbc7297c54b9d921b29ce11d8077f_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6dcbc7297c54b9d921b29ce11d8077f_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:28,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44bfe8b3a6f046beb8bd50499618e65c, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:28,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44bfe8b3a6f046beb8bd50499618e65c is 175, key is test_row_0/A:col10/1732224328606/Put/seqid=0 2024-11-21T21:25:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:25:28,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742192_1368 (size=30955) 2024-11-21T21:25:28,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:28,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:28,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:28,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224388811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224388811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224388819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224388819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224388821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:28,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:28,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:28,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:28,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:28,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:25:29,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224389029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224389030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224389031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224389031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224389040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44bfe8b3a6f046beb8bd50499618e65c 2024-11-21T21:25:29,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/ae61227181024b96b11ebc044632da19 is 50, key is test_row_0/B:col10/1732224328606/Put/seqid=0 2024-11-21T21:25:29,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742193_1369 (size=12001) 2024-11-21T21:25:29,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/ae61227181024b96b11ebc044632da19 2024-11-21T21:25:29,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f5c3819ac6f54f589fe9f1be38988a9e is 50, key is test_row_0/C:col10/1732224328606/Put/seqid=0 2024-11-21T21:25:29,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:29,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:29,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:25:29,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742194_1370 (size=12001) 2024-11-21T21:25:29,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f5c3819ac6f54f589fe9f1be38988a9e 2024-11-21T21:25:29,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44bfe8b3a6f046beb8bd50499618e65c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44bfe8b3a6f046beb8bd50499618e65c 2024-11-21T21:25:29,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44bfe8b3a6f046beb8bd50499618e65c, entries=150, sequenceid=15, filesize=30.2 K 2024-11-21T21:25:29,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/ae61227181024b96b11ebc044632da19 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/ae61227181024b96b11ebc044632da19 2024-11-21T21:25:29,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/ae61227181024b96b11ebc044632da19, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T21:25:29,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f5c3819ac6f54f589fe9f1be38988a9e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f5c3819ac6f54f589fe9f1be38988a9e 2024-11-21T21:25:29,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f5c3819ac6f54f589fe9f1be38988a9e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T21:25:29,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 58ad0d609f8c768bc6016e57e5d831ac in 738ms, sequenceid=15, compaction requested=false 2024-11-21T21:25:29,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:29,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T21:25:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224389363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224389365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224389365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f4867da266e9462eaa9e68c6cfe32a23_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224328690/Put/seqid=0 2024-11-21T21:25:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224389375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224389375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742195_1371 (size=17034) 2024-11-21T21:25:29,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,424 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f4867da266e9462eaa9e68c6cfe32a23_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f4867da266e9462eaa9e68c6cfe32a23_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:29,427 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/42e460b5ea0e48e08d29ffe25a8c8cc8, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:29,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/42e460b5ea0e48e08d29ffe25a8c8cc8 is 175, key is test_row_0/A:col10/1732224328690/Put/seqid=0 2024-11-21T21:25:29,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:29,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742196_1372 (size=48139) 2024-11-21T21:25:29,449 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/42e460b5ea0e48e08d29ffe25a8c8cc8 2024-11-21T21:25:29,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/a8db69c31b7a4965bb02182a71bd890d is 50, key is test_row_0/B:col10/1732224328690/Put/seqid=0 2024-11-21T21:25:29,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224389480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224389480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224389480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742197_1373 (size=12001) 2024-11-21T21:25:29,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224389499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224389500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224389691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224389699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224389701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224389711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224389720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:29,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:25:29,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/a8db69c31b7a4965bb02182a71bd890d 2024-11-21T21:25:29,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:29,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:29,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:29,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/b27a050854af46eaa860e93a15ed1f68 is 50, key is test_row_0/C:col10/1732224328690/Put/seqid=0 2024-11-21T21:25:29,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:29,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742198_1374 (size=12001) 2024-11-21T21:25:29,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/b27a050854af46eaa860e93a15ed1f68 2024-11-21T21:25:29,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/42e460b5ea0e48e08d29ffe25a8c8cc8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/42e460b5ea0e48e08d29ffe25a8c8cc8 2024-11-21T21:25:29,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/42e460b5ea0e48e08d29ffe25a8c8cc8, entries=250, sequenceid=42, filesize=47.0 K 2024-11-21T21:25:29,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/a8db69c31b7a4965bb02182a71bd890d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a8db69c31b7a4965bb02182a71bd890d 2024-11-21T21:25:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a8db69c31b7a4965bb02182a71bd890d, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T21:25:29,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/b27a050854af46eaa860e93a15ed1f68 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/b27a050854af46eaa860e93a15ed1f68 2024-11-21T21:25:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/b27a050854af46eaa860e93a15ed1f68, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T21:25:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 58ad0d609f8c768bc6016e57e5d831ac in 636ms, sequenceid=42, compaction requested=false 2024-11-21T21:25:29,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T21:25:30,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:30,061 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T21:25:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121634ece2ef08a44c4abc433f36e32ec8a_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224329363/Put/seqid=0 2024-11-21T21:25:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742199_1375 (size=9714) 2024-11-21T21:25:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,118 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121634ece2ef08a44c4abc433f36e32ec8a_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121634ece2ef08a44c4abc433f36e32ec8a_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:30,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/637ccf8b96a243e7949b7f1bc924b15e, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:30,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/637ccf8b96a243e7949b7f1bc924b15e is 175, key is test_row_0/A:col10/1732224329363/Put/seqid=0 2024-11-21T21:25:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742200_1376 (size=22361) 2024-11-21T21:25:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,177 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/637ccf8b96a243e7949b7f1bc924b15e 2024-11-21T21:25:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/3590f22abe444b8b9ef4777fc12d0128 is 50, key is test_row_0/B:col10/1732224329363/Put/seqid=0 2024-11-21T21:25:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742201_1377 (size=9657) 2024-11-21T21:25:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224390352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224390364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224390373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224390373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224390374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224390480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224390491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224390500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224390500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224390500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,651 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/3590f22abe444b8b9ef4777fc12d0128 2024-11-21T21:25:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/9171d837cba6493bbe3f283b6c1496e7 is 50, key is test_row_0/C:col10/1732224329363/Put/seqid=0 2024-11-21T21:25:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224390699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224390700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742202_1378 (size=9657) 2024-11-21T21:25:30,717 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/9171d837cba6493bbe3f283b6c1496e7 2024-11-21T21:25:30,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/637ccf8b96a243e7949b7f1bc924b15e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/637ccf8b96a243e7949b7f1bc924b15e 2024-11-21T21:25:30,726 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/637ccf8b96a243e7949b7f1bc924b15e, entries=100, sequenceid=51, filesize=21.8 K 2024-11-21T21:25:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/3590f22abe444b8b9ef4777fc12d0128 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3590f22abe444b8b9ef4777fc12d0128 2024-11-21T21:25:30,732 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3590f22abe444b8b9ef4777fc12d0128, entries=100, sequenceid=51, filesize=9.4 K 2024-11-21T21:25:30,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/9171d837cba6493bbe3f283b6c1496e7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9171d837cba6493bbe3f283b6c1496e7 2024-11-21T21:25:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224390729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224390729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224390730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:30,738 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9171d837cba6493bbe3f283b6c1496e7, entries=100, sequenceid=51, filesize=9.4 K 2024-11-21T21:25:30,741 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 58ad0d609f8c768bc6016e57e5d831ac in 680ms, sequenceid=51, compaction requested=true 2024-11-21T21:25:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-21T21:25:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-21T21:25:30,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-21T21:25:30,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0830 sec 2024-11-21T21:25:30,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.0880 sec 2024-11-21T21:25:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:25:30,810 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:30,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-21T21:25:30,812 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T21:25:30,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:30,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T21:25:30,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-21T21:25:30,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:30,966 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T21:25:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112193eda7937596498da2bf6b3cd3b2ac7f_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224330372/Put/seqid=0 2024-11-21T21:25:31,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742203_1379 (size=12154) 2024-11-21T21:25:31,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,020 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112193eda7937596498da2bf6b3cd3b2ac7f_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112193eda7937596498da2bf6b3cd3b2ac7f_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/77cf3db2b9c04cfb94538f379fa4b71c, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/77cf3db2b9c04cfb94538f379fa4b71c is 175, key is test_row_0/A:col10/1732224330372/Put/seqid=0 2024-11-21T21:25:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224391047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224391049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224391049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224391049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742204_1380 (size=30955) 2024-11-21T21:25:31,075 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/77cf3db2b9c04cfb94538f379fa4b71c 2024-11-21T21:25:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224391064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/dd42350630384033a2f6131fd2acf777 is 50, key is test_row_0/B:col10/1732224330372/Put/seqid=0 2024-11-21T21:25:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T21:25:31,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742205_1381 (size=12001) 2024-11-21T21:25:31,148 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/dd42350630384033a2f6131fd2acf777 2024-11-21T21:25:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/0e3c57d90d574a399e93776b72abced8 is 50, key is test_row_0/C:col10/1732224330372/Put/seqid=0 2024-11-21T21:25:31,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224391170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224391170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224391189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742206_1382 (size=12001) 2024-11-21T21:25:31,211 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/0e3c57d90d574a399e93776b72abced8 2024-11-21T21:25:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/77cf3db2b9c04cfb94538f379fa4b71c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/77cf3db2b9c04cfb94538f379fa4b71c 2024-11-21T21:25:31,229 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/77cf3db2b9c04cfb94538f379fa4b71c, entries=150, sequenceid=78, filesize=30.2 K 2024-11-21T21:25:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/dd42350630384033a2f6131fd2acf777 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/dd42350630384033a2f6131fd2acf777 2024-11-21T21:25:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,238 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/dd42350630384033a2f6131fd2acf777, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T21:25:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/0e3c57d90d574a399e93776b72abced8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/0e3c57d90d574a399e93776b72abced8 2024-11-21T21:25:31,251 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:25:31,255 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/0e3c57d90d574a399e93776b72abced8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T21:25:31,257 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 58ad0d609f8c768bc6016e57e5d831ac in 291ms, sequenceid=78, compaction requested=true 2024-11-21T21:25:31,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:31,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:31,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-21T21:25:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-21T21:25:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-21T21:25:31,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 446 msec 2024-11-21T21:25:31,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 452 msec 2024-11-21T21:25:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T21:25:31,440 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-21T21:25:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T21:25:31,442 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,443 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:31,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:31,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:25:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b810027bfd6e429db6b00575020fe216_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224331467/Put/seqid=0 2024-11-21T21:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T21:25:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742207_1383 (size=14594) 2024-11-21T21:25:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,575 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b810027bfd6e429db6b00575020fe216_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b810027bfd6e429db6b00575020fe216_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,576 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/866f8358cf1443339c3a56f4c7e22a08, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/866f8358cf1443339c3a56f4c7e22a08 is 175, key is test_row_0/A:col10/1732224331467/Put/seqid=0 2024-11-21T21:25:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T21:25:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:31,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742208_1384 (size=39545) 2024-11-21T21:25:31,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/866f8358cf1443339c3a56f4c7e22a08 2024-11-21T21:25:31,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/f3b4fbba1a9f4aff8bb277c51e266260 is 50, key is test_row_0/B:col10/1732224331467/Put/seqid=0 2024-11-21T21:25:31,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224391663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742209_1385 (size=9657) 2024-11-21T21:25:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224391666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224391668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224391668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224391668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T21:25:31,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:31,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:31,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:31,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T21:25:31,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224391780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224391789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224391789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224391790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224391790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:31,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T21:25:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:31,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:32,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224392000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224392010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224392010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224392010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224392010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T21:25:32,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:32,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T21:25:32,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/f3b4fbba1a9f4aff8bb277c51e266260 2024-11-21T21:25:32,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/501c7da17e6b443b983b3913467a1f01 is 50, key is test_row_0/C:col10/1732224331467/Put/seqid=0 2024-11-21T21:25:32,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742210_1386 (size=9657) 2024-11-21T21:25:32,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/501c7da17e6b443b983b3913467a1f01 2024-11-21T21:25:32,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/866f8358cf1443339c3a56f4c7e22a08 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/866f8358cf1443339c3a56f4c7e22a08 2024-11-21T21:25:32,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/866f8358cf1443339c3a56f4c7e22a08, entries=200, sequenceid=89, filesize=38.6 K 2024-11-21T21:25:32,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/f3b4fbba1a9f4aff8bb277c51e266260 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3b4fbba1a9f4aff8bb277c51e266260 2024-11-21T21:25:32,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3b4fbba1a9f4aff8bb277c51e266260, entries=100, sequenceid=89, filesize=9.4 K 2024-11-21T21:25:32,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/501c7da17e6b443b983b3913467a1f01 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/501c7da17e6b443b983b3913467a1f01 2024-11-21T21:25:32,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/501c7da17e6b443b983b3913467a1f01, entries=100, sequenceid=89, filesize=9.4 K 2024-11-21T21:25:32,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 58ad0d609f8c768bc6016e57e5d831ac in 713ms, sequenceid=89, compaction requested=true 2024-11-21T21:25:32,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:32,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:32,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:32,181 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T21:25:32,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:32,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:32,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:32,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:25:32,182 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T21:25:32,183 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 171955 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T21:25:32,183 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/A is initiating minor compaction (all files) 2024-11-21T21:25:32,183 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/A in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,183 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44bfe8b3a6f046beb8bd50499618e65c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/42e460b5ea0e48e08d29ffe25a8c8cc8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/637ccf8b96a243e7949b7f1bc924b15e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/77cf3db2b9c04cfb94538f379fa4b71c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/866f8358cf1443339c3a56f4c7e22a08] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=167.9 K 2024-11-21T21:25:32,183 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,183 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44bfe8b3a6f046beb8bd50499618e65c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/42e460b5ea0e48e08d29ffe25a8c8cc8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/637ccf8b96a243e7949b7f1bc924b15e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/77cf3db2b9c04cfb94538f379fa4b71c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/866f8358cf1443339c3a56f4c7e22a08] 2024-11-21T21:25:32,184 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 44bfe8b3a6f046beb8bd50499618e65c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732224328600 2024-11-21T21:25:32,184 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 42e460b5ea0e48e08d29ffe25a8c8cc8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732224328690 2024-11-21T21:25:32,185 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 637ccf8b96a243e7949b7f1bc924b15e, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224329363 2024-11-21T21:25:32,185 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 77cf3db2b9c04cfb94538f379fa4b71c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732224330362 2024-11-21T21:25:32,185 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 55317 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T21:25:32,185 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/B is initiating minor compaction (all files) 2024-11-21T21:25:32,185 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/B in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,185 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/ae61227181024b96b11ebc044632da19, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a8db69c31b7a4965bb02182a71bd890d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3590f22abe444b8b9ef4777fc12d0128, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/dd42350630384033a2f6131fd2acf777, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3b4fbba1a9f4aff8bb277c51e266260] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=54.0 K 2024-11-21T21:25:32,186 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae61227181024b96b11ebc044632da19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732224328600 2024-11-21T21:25:32,186 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 866f8358cf1443339c3a56f4c7e22a08, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732224331043 2024-11-21T21:25:32,186 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8db69c31b7a4965bb02182a71bd890d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732224328690 2024-11-21T21:25:32,187 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3590f22abe444b8b9ef4777fc12d0128, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224329363 2024-11-21T21:25:32,187 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd42350630384033a2f6131fd2acf777, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732224330362 2024-11-21T21:25:32,188 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3b4fbba1a9f4aff8bb277c51e266260, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732224331466 2024-11-21T21:25:32,210 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:32,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T21:25:32,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,215 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:25:32,235 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:32,235 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/20a9f01a942548c7bda34f7d0550ac3d is 50, key is test_row_0/B:col10/1732224331467/Put/seqid=0 2024-11-21T21:25:32,250 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f8bcf0810ab844e5bfc4f64d98a25a90_58ad0d609f8c768bc6016e57e5d831ac store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:32,254 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f8bcf0810ab844e5bfc4f64d98a25a90_58ad0d609f8c768bc6016e57e5d831ac, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:32,254 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f8bcf0810ab844e5bfc4f64d98a25a90_58ad0d609f8c768bc6016e57e5d831ac because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4cbea75e7f3456db0db9a02c12024d0_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224331663/Put/seqid=0 2024-11-21T21:25:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:32,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:32,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742211_1387 (size=12173) 2024-11-21T21:25:32,358 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/20a9f01a942548c7bda34f7d0550ac3d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/20a9f01a942548c7bda34f7d0550ac3d 2024-11-21T21:25:32,364 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/B of 58ad0d609f8c768bc6016e57e5d831ac into 20a9f01a942548c7bda34f7d0550ac3d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:32,364 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:32,364 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/B, priority=11, startTime=1732224332181; duration=0sec 2024-11-21T21:25:32,364 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:32,364 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:B 2024-11-21T21:25:32,364 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T21:25:32,366 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 55317 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T21:25:32,366 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/C is initiating minor compaction (all files) 2024-11-21T21:25:32,366 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/C in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,366 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f5c3819ac6f54f589fe9f1be38988a9e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/b27a050854af46eaa860e93a15ed1f68, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9171d837cba6493bbe3f283b6c1496e7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/0e3c57d90d574a399e93776b72abced8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/501c7da17e6b443b983b3913467a1f01] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=54.0 K 2024-11-21T21:25:32,366 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c3819ac6f54f589fe9f1be38988a9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732224328600 2024-11-21T21:25:32,367 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27a050854af46eaa860e93a15ed1f68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732224328690 2024-11-21T21:25:32,367 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9171d837cba6493bbe3f283b6c1496e7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224329363 2024-11-21T21:25:32,367 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e3c57d90d574a399e93776b72abced8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732224330362 2024-11-21T21:25:32,368 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 501c7da17e6b443b983b3913467a1f01, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732224331466 2024-11-21T21:25:32,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742212_1388 (size=4469) 2024-11-21T21:25:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224392351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,380 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#A#compaction#327 average throughput is 0.14 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224392351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,381 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/277d89bdcef243fe99857b5b2f9140a2 is 175, key is test_row_0/A:col10/1732224331467/Put/seqid=0 2024-11-21T21:25:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224392375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742213_1389 (size=12154) 2024-11-21T21:25:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:32,399 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#C#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:32,399 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f3136832134641519ba25ba2bc8585b1 is 50, key is test_row_0/C:col10/1732224331467/Put/seqid=0 2024-11-21T21:25:32,403 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4cbea75e7f3456db0db9a02c12024d0_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4cbea75e7f3456db0db9a02c12024d0_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:32,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224392380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224392380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/18d6af7d5ae5471582bd4cc159d0c3ae, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:32,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/18d6af7d5ae5471582bd4cc159d0c3ae is 175, key is test_row_0/A:col10/1732224331663/Put/seqid=0 2024-11-21T21:25:32,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742214_1390 (size=31234) 2024-11-21T21:25:32,432 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/277d89bdcef243fe99857b5b2f9140a2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/277d89bdcef243fe99857b5b2f9140a2 2024-11-21T21:25:32,438 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/A of 58ad0d609f8c768bc6016e57e5d831ac into 277d89bdcef243fe99857b5b2f9140a2(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:32,438 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:32,438 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/A, priority=11, startTime=1732224332181; duration=0sec 2024-11-21T21:25:32,439 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:32,439 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:A 2024-11-21T21:25:32,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742215_1391 (size=12173) 2024-11-21T21:25:32,446 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f3136832134641519ba25ba2bc8585b1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f3136832134641519ba25ba2bc8585b1 2024-11-21T21:25:32,454 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/C of 58ad0d609f8c768bc6016e57e5d831ac into f3136832134641519ba25ba2bc8585b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:32,455 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:32,455 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/C, priority=11, startTime=1732224332182; duration=0sec 2024-11-21T21:25:32,455 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:32,455 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:C 2024-11-21T21:25:32,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742216_1392 (size=30955) 2024-11-21T21:25:32,468 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/18d6af7d5ae5471582bd4cc159d0c3ae 2024-11-21T21:25:32,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/837d7eff133644a28989fd5f0b693404 is 50, key is test_row_0/B:col10/1732224331663/Put/seqid=0 2024-11-21T21:25:32,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742217_1393 (size=12001) 2024-11-21T21:25:32,482 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/837d7eff133644a28989fd5f0b693404 2024-11-21T21:25:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/8acedc4ec00d44a190815d6bb03951a8 is 50, key is test_row_0/C:col10/1732224331663/Put/seqid=0 2024-11-21T21:25:32,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224392489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224392490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742218_1394 (size=12001) 2024-11-21T21:25:32,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224392499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224392510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224392510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T21:25:32,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224392700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224392700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224392709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224392720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224392720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:32,903 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/8acedc4ec00d44a190815d6bb03951a8 2024-11-21T21:25:32,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/18d6af7d5ae5471582bd4cc159d0c3ae as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/18d6af7d5ae5471582bd4cc159d0c3ae 2024-11-21T21:25:32,922 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/18d6af7d5ae5471582bd4cc159d0c3ae, entries=150, sequenceid=114, filesize=30.2 K 2024-11-21T21:25:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/837d7eff133644a28989fd5f0b693404 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/837d7eff133644a28989fd5f0b693404 2024-11-21T21:25:32,931 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/837d7eff133644a28989fd5f0b693404, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:25:32,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/8acedc4ec00d44a190815d6bb03951a8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/8acedc4ec00d44a190815d6bb03951a8 2024-11-21T21:25:32,936 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/8acedc4ec00d44a190815d6bb03951a8, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:25:32,938 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 58ad0d609f8c768bc6016e57e5d831ac in 723ms, sequenceid=114, compaction requested=false 2024-11-21T21:25:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-21T21:25:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-21T21:25:32,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-21T21:25:32,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-21T21:25:32,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5060 sec 2024-11-21T21:25:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:33,016 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:33,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:25:33,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217872de3ad45741ac85d9a6167cbeea64_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224333015/Put/seqid=0 2024-11-21T21:25:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742219_1395 (size=14644) 2024-11-21T21:25:33,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224393072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224393073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224393075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224393077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224393077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224393189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224393201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224393201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224393201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224393201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224393400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224393409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224393409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224393410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224393410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,441 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:33,444 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217872de3ad45741ac85d9a6167cbeea64_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217872de3ad45741ac85d9a6167cbeea64_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:33,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/dab4762355214ab7ad505d1c2bb308a9, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:33,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/dab4762355214ab7ad505d1c2bb308a9 is 175, key is test_row_0/A:col10/1732224333015/Put/seqid=0 2024-11-21T21:25:33,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742220_1396 (size=39599) 2024-11-21T21:25:33,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/dab4762355214ab7ad505d1c2bb308a9 2024-11-21T21:25:33,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/d2eaa87024274a0687cab28bb07c4597 is 50, key is test_row_0/B:col10/1732224333015/Put/seqid=0 2024-11-21T21:25:33,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742221_1397 (size=12051) 2024-11-21T21:25:33,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/d2eaa87024274a0687cab28bb07c4597 2024-11-21T21:25:33,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f40ab871923b4e8e89fc3a5d2993a17f is 50, key is test_row_0/C:col10/1732224333015/Put/seqid=0 2024-11-21T21:25:33,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742222_1398 (size=12051) 2024-11-21T21:25:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T21:25:33,589 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:33,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-21T21:25:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T21:25:33,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:33,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:33,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T21:25:33,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224393709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224393730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224393730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224393730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224393731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T21:25:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:33,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:33,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T21:25:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:33,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:33,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T21:25:33,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f40ab871923b4e8e89fc3a5d2993a17f 2024-11-21T21:25:33,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/dab4762355214ab7ad505d1c2bb308a9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/dab4762355214ab7ad505d1c2bb308a9 2024-11-21T21:25:33,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/dab4762355214ab7ad505d1c2bb308a9, entries=200, sequenceid=129, filesize=38.7 K 2024-11-21T21:25:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/d2eaa87024274a0687cab28bb07c4597 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d2eaa87024274a0687cab28bb07c4597 2024-11-21T21:25:33,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d2eaa87024274a0687cab28bb07c4597, entries=150, sequenceid=129, filesize=11.8 K 2024-11-21T21:25:33,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/f40ab871923b4e8e89fc3a5d2993a17f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f40ab871923b4e8e89fc3a5d2993a17f 2024-11-21T21:25:33,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f40ab871923b4e8e89fc3a5d2993a17f, entries=150, sequenceid=129, filesize=11.8 K 2024-11-21T21:25:33,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 58ad0d609f8c768bc6016e57e5d831ac in 928ms, sequenceid=129, compaction requested=true 2024-11-21T21:25:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:33,945 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:33,945 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:33,949 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:33,949 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/B is initiating minor compaction (all files) 2024-11-21T21:25:33,949 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/B in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:33,950 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/20a9f01a942548c7bda34f7d0550ac3d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/837d7eff133644a28989fd5f0b693404, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d2eaa87024274a0687cab28bb07c4597] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=35.4 K 2024-11-21T21:25:33,950 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101788 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:33,950 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/A is initiating minor compaction (all files) 2024-11-21T21:25:33,950 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/A in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:33,950 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a9f01a942548c7bda34f7d0550ac3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732224330362 2024-11-21T21:25:33,950 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/277d89bdcef243fe99857b5b2f9140a2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/18d6af7d5ae5471582bd4cc159d0c3ae, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/dab4762355214ab7ad505d1c2bb308a9] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=99.4 K 2024-11-21T21:25:33,950 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:33,950 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/277d89bdcef243fe99857b5b2f9140a2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/18d6af7d5ae5471582bd4cc159d0c3ae, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/dab4762355214ab7ad505d1c2bb308a9] 2024-11-21T21:25:33,950 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 837d7eff133644a28989fd5f0b693404, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224331661 2024-11-21T21:25:33,950 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 277d89bdcef243fe99857b5b2f9140a2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732224330362 2024-11-21T21:25:33,950 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting d2eaa87024274a0687cab28bb07c4597, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732224332347 2024-11-21T21:25:33,951 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18d6af7d5ae5471582bd4cc159d0c3ae, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224331661 2024-11-21T21:25:33,951 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting dab4762355214ab7ad505d1c2bb308a9, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732224332347 2024-11-21T21:25:33,974 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:33,975 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/5cf3b20b276a41c29dcd11a3efb4ccea is 50, key is test_row_0/B:col10/1732224333015/Put/seqid=0 2024-11-21T21:25:33,983 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:34,001 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a9b31386049046a3909063eef8a46a12_58ad0d609f8c768bc6016e57e5d831ac store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:34,003 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a9b31386049046a3909063eef8a46a12_58ad0d609f8c768bc6016e57e5d831ac, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:34,003 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a9b31386049046a3909063eef8a46a12_58ad0d609f8c768bc6016e57e5d831ac because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:34,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742223_1399 (size=12325) 2024-11-21T21:25:34,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742224_1400 (size=4469) 2024-11-21T21:25:34,044 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#A#compaction#337 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:34,045 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/ee8ad6944b6e48a68a6af892ee73ebf0 is 175, key is test_row_0/A:col10/1732224333015/Put/seqid=0 2024-11-21T21:25:34,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T21:25:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:34,063 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:25:34,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742225_1401 (size=31279) 2024-11-21T21:25:34,076 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/ee8ad6944b6e48a68a6af892ee73ebf0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ee8ad6944b6e48a68a6af892ee73ebf0 2024-11-21T21:25:34,084 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/A of 58ad0d609f8c768bc6016e57e5d831ac into ee8ad6944b6e48a68a6af892ee73ebf0(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:34,084 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:34,084 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/A, priority=13, startTime=1732224333945; duration=0sec 2024-11-21T21:25:34,084 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:34,084 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:A 2024-11-21T21:25:34,084 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c5a513ef8a594c6f9e9dd4cc63a10af4_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224333073/Put/seqid=0 2024-11-21T21:25:34,088 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:34,088 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/C is initiating minor compaction (all files) 2024-11-21T21:25:34,088 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/C in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:34,088 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f3136832134641519ba25ba2bc8585b1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/8acedc4ec00d44a190815d6bb03951a8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f40ab871923b4e8e89fc3a5d2993a17f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=35.4 K 2024-11-21T21:25:34,089 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3136832134641519ba25ba2bc8585b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732224330362 2024-11-21T21:25:34,089 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8acedc4ec00d44a190815d6bb03951a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224331661 2024-11-21T21:25:34,089 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f40ab871923b4e8e89fc3a5d2993a17f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732224332347 2024-11-21T21:25:34,098 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:34,099 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/c79042f30c6b4362ad4342d9d3282f58 is 50, key is test_row_0/C:col10/1732224333015/Put/seqid=0 2024-11-21T21:25:34,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742226_1402 (size=12325) 2024-11-21T21:25:34,132 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/c79042f30c6b4362ad4342d9d3282f58 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c79042f30c6b4362ad4342d9d3282f58 2024-11-21T21:25:34,138 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/C of 58ad0d609f8c768bc6016e57e5d831ac into c79042f30c6b4362ad4342d9d3282f58(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:34,138 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:34,138 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/C, priority=13, startTime=1732224333945; duration=0sec 2024-11-21T21:25:34,139 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:34,139 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:C 2024-11-21T21:25:34,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742227_1403 (size=12304) 2024-11-21T21:25:34,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:34,151 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c5a513ef8a594c6f9e9dd4cc63a10af4_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c5a513ef8a594c6f9e9dd4cc63a10af4_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:34,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/3323368efd8b475f8085edeccd1765bc, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:34,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/3323368efd8b475f8085edeccd1765bc is 175, key is test_row_0/A:col10/1732224333073/Put/seqid=0 2024-11-21T21:25:34,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742228_1404 (size=31105) 2024-11-21T21:25:34,214 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/3323368efd8b475f8085edeccd1765bc 2024-11-21T21:25:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T21:25:34,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/e509032b7dc14a0395550928776ad2a2 is 50, key is test_row_0/B:col10/1732224333073/Put/seqid=0 2024-11-21T21:25:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:34,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:34,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742229_1405 (size=12151) 2024-11-21T21:25:34,258 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/e509032b7dc14a0395550928776ad2a2 2024-11-21T21:25:34,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/2095c84040d744c399f892ef9ea63f53 is 50, key is test_row_0/C:col10/1732224333073/Put/seqid=0 2024-11-21T21:25:34,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224394262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224394272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224394273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224394276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224394276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742230_1406 (size=12151) 2024-11-21T21:25:34,292 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/2095c84040d744c399f892ef9ea63f53 2024-11-21T21:25:34,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/3323368efd8b475f8085edeccd1765bc as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/3323368efd8b475f8085edeccd1765bc 2024-11-21T21:25:34,303 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/3323368efd8b475f8085edeccd1765bc, entries=150, sequenceid=153, filesize=30.4 K 2024-11-21T21:25:34,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/e509032b7dc14a0395550928776ad2a2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e509032b7dc14a0395550928776ad2a2 2024-11-21T21:25:34,308 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e509032b7dc14a0395550928776ad2a2, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T21:25:34,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/2095c84040d744c399f892ef9ea63f53 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/2095c84040d744c399f892ef9ea63f53 2024-11-21T21:25:34,325 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/2095c84040d744c399f892ef9ea63f53, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T21:25:34,329 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 58ad0d609f8c768bc6016e57e5d831ac in 266ms, sequenceid=153, compaction requested=false 2024-11-21T21:25:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-21T21:25:34,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-21T21:25:34,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-21T21:25:34,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 739 msec 2024-11-21T21:25:34,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 744 msec 2024-11-21T21:25:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:34,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:25:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:34,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d514e075761d4a87bec88571d97ee203_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224334255/Put/seqid=0 2024-11-21T21:25:34,431 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/5cf3b20b276a41c29dcd11a3efb4ccea as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/5cf3b20b276a41c29dcd11a3efb4ccea 2024-11-21T21:25:34,495 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/B of 58ad0d609f8c768bc6016e57e5d831ac into 5cf3b20b276a41c29dcd11a3efb4ccea(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:34,495 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:34,495 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/B, priority=13, startTime=1732224333945; duration=0sec 2024-11-21T21:25:34,495 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:34,495 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:B 2024-11-21T21:25:34,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742231_1407 (size=14794) 2024-11-21T21:25:34,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:34,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d514e075761d4a87bec88571d97ee203_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d514e075761d4a87bec88571d97ee203_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:34,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/d03986302a2e4b41ba22ed329232b5ad, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:34,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/d03986302a2e4b41ba22ed329232b5ad is 175, key is test_row_0/A:col10/1732224334255/Put/seqid=0 2024-11-21T21:25:34,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742232_1408 (size=39749) 2024-11-21T21:25:34,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224394536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224394537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224394537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224394538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224394555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224394660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224394660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224394669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224394669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224394680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T21:25:34,732 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:34,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-21T21:25:34,734 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:34,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:34,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:25:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:25:34,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224394880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224394880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:25:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:34,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:34,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224394891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224394891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224394900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:34,945 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/d03986302a2e4b41ba22ed329232b5ad 2024-11-21T21:25:34,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/55281abf3f4f4e2e956e308aa22755c7 is 50, key is test_row_0/B:col10/1732224334255/Put/seqid=0 2024-11-21T21:25:34,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742233_1409 (size=12151) 2024-11-21T21:25:35,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:25:35,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:35,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:25:35,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224395200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224395200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224395200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:25:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:35,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224395209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224395219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:25:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:35,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/55281abf3f4f4e2e956e308aa22755c7 2024-11-21T21:25:35,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/cd1cccf4f8c3458c9b37d97b43146f25 is 50, key is test_row_0/C:col10/1732224334255/Put/seqid=0 2024-11-21T21:25:35,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742234_1410 (size=12151) 2024-11-21T21:25:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:25:35,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:25:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:25:35,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:35,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224395709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224395709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224395709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224395719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224395730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:35,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/cd1cccf4f8c3458c9b37d97b43146f25 2024-11-21T21:25:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/d03986302a2e4b41ba22ed329232b5ad as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/d03986302a2e4b41ba22ed329232b5ad 2024-11-21T21:25:35,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/d03986302a2e4b41ba22ed329232b5ad, entries=200, sequenceid=168, filesize=38.8 K 2024-11-21T21:25:35,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/55281abf3f4f4e2e956e308aa22755c7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/55281abf3f4f4e2e956e308aa22755c7 2024-11-21T21:25:35,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/55281abf3f4f4e2e956e308aa22755c7, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T21:25:35,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/cd1cccf4f8c3458c9b37d97b43146f25 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/cd1cccf4f8c3458c9b37d97b43146f25 2024-11-21T21:25:35,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/cd1cccf4f8c3458c9b37d97b43146f25, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T21:25:35,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 58ad0d609f8c768bc6016e57e5d831ac in 1398ms, sequenceid=168, compaction requested=true 2024-11-21T21:25:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:35,784 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:35,784 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:35,785 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102133 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:35,785 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:35,785 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/A is initiating minor compaction (all files) 2024-11-21T21:25:35,785 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/B is initiating minor compaction (all files) 2024-11-21T21:25:35,785 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/B in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,785 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/A in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,785 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ee8ad6944b6e48a68a6af892ee73ebf0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/3323368efd8b475f8085edeccd1765bc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/d03986302a2e4b41ba22ed329232b5ad] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=99.7 K 2024-11-21T21:25:35,785 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/5cf3b20b276a41c29dcd11a3efb4ccea, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e509032b7dc14a0395550928776ad2a2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/55281abf3f4f4e2e956e308aa22755c7] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=35.8 K 2024-11-21T21:25:35,785 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,785 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ee8ad6944b6e48a68a6af892ee73ebf0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/3323368efd8b475f8085edeccd1765bc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/d03986302a2e4b41ba22ed329232b5ad] 2024-11-21T21:25:35,785 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf3b20b276a41c29dcd11a3efb4ccea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732224332347 2024-11-21T21:25:35,785 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8ad6944b6e48a68a6af892ee73ebf0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732224332347 2024-11-21T21:25:35,786 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e509032b7dc14a0395550928776ad2a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732224333073 2024-11-21T21:25:35,786 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3323368efd8b475f8085edeccd1765bc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732224333073 2024-11-21T21:25:35,786 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 55281abf3f4f4e2e956e308aa22755c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732224334255 2024-11-21T21:25:35,786 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03986302a2e4b41ba22ed329232b5ad, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732224334255 2024-11-21T21:25:35,792 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:35,793 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/84bb5c8d5ac1465f880c7cb8bcf7bc27 is 50, key is test_row_0/B:col10/1732224334255/Put/seqid=0 2024-11-21T21:25:35,794 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:35,796 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121215c26e5ebf24e66a74dea08f874d423_58ad0d609f8c768bc6016e57e5d831ac store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:35,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742235_1411 (size=12527) 2024-11-21T21:25:35,798 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121215c26e5ebf24e66a74dea08f874d423_58ad0d609f8c768bc6016e57e5d831ac, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:35,798 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121215c26e5ebf24e66a74dea08f874d423_58ad0d609f8c768bc6016e57e5d831ac because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:35,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742236_1412 (size=4469) 2024-11-21T21:25:35,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:25:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:35,819 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:25:35,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bbafa11903244861a54960e0f2287e09_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224334550/Put/seqid=0 2024-11-21T21:25:35,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742237_1413 (size=12304) 2024-11-21T21:25:35,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:35,832 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bbafa11903244861a54960e0f2287e09_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bbafa11903244861a54960e0f2287e09_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:35,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/0fe431059e7145dea4997abca1425f3b, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:35,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/0fe431059e7145dea4997abca1425f3b is 175, key is test_row_0/A:col10/1732224334550/Put/seqid=0 2024-11-21T21:25:35,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742238_1414 (size=31105) 2024-11-21T21:25:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:25:36,205 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/84bb5c8d5ac1465f880c7cb8bcf7bc27 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/84bb5c8d5ac1465f880c7cb8bcf7bc27 2024-11-21T21:25:36,205 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:36,206 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/2e776e3cb2114547b7bff924402d98b4 is 175, key is test_row_0/A:col10/1732224334255/Put/seqid=0 2024-11-21T21:25:36,212 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/B of 58ad0d609f8c768bc6016e57e5d831ac into 84bb5c8d5ac1465f880c7cb8bcf7bc27(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:36,213 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:36,213 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/B, priority=13, startTime=1732224335784; duration=0sec 2024-11-21T21:25:36,213 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:36,213 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:B 2024-11-21T21:25:36,213 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:36,214 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:36,214 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/C is initiating minor compaction (all files) 2024-11-21T21:25:36,214 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/C in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:36,214 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c79042f30c6b4362ad4342d9d3282f58, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/2095c84040d744c399f892ef9ea63f53, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/cd1cccf4f8c3458c9b37d97b43146f25] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=35.8 K 2024-11-21T21:25:36,215 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c79042f30c6b4362ad4342d9d3282f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732224332347 2024-11-21T21:25:36,215 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2095c84040d744c399f892ef9ea63f53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732224333073 2024-11-21T21:25:36,217 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1cccf4f8c3458c9b37d97b43146f25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732224334255 2024-11-21T21:25:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742239_1415 (size=31481) 2024-11-21T21:25:36,232 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#C#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:36,233 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/463c8da8e0fb4243853329ff6a42b97d is 50, key is test_row_0/C:col10/1732224334255/Put/seqid=0 2024-11-21T21:25:36,237 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/0fe431059e7145dea4997abca1425f3b 2024-11-21T21:25:36,238 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/2e776e3cb2114547b7bff924402d98b4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/2e776e3cb2114547b7bff924402d98b4 2024-11-21T21:25:36,246 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/A of 58ad0d609f8c768bc6016e57e5d831ac into 2e776e3cb2114547b7bff924402d98b4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:36,246 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:36,246 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/A, priority=13, startTime=1732224335784; duration=0sec 2024-11-21T21:25:36,246 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:36,246 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:A 2024-11-21T21:25:36,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/3f93f033ac2c4625b44ccf1fcfbb90b3 is 50, key is test_row_0/B:col10/1732224334550/Put/seqid=0 2024-11-21T21:25:36,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742240_1416 (size=12527) 2024-11-21T21:25:36,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742241_1417 (size=12151) 2024-11-21T21:25:36,289 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/463c8da8e0fb4243853329ff6a42b97d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/463c8da8e0fb4243853329ff6a42b97d 2024-11-21T21:25:36,293 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/3f93f033ac2c4625b44ccf1fcfbb90b3 2024-11-21T21:25:36,305 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/C of 58ad0d609f8c768bc6016e57e5d831ac into 463c8da8e0fb4243853329ff6a42b97d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:36,305 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:36,305 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/C, priority=13, startTime=1732224335784; duration=0sec 2024-11-21T21:25:36,305 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:36,305 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:C 2024-11-21T21:25:36,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/ba929c69259341e3839018159ddd7b89 is 50, key is test_row_0/C:col10/1732224334550/Put/seqid=0 2024-11-21T21:25:36,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742242_1418 (size=12151) 2024-11-21T21:25:36,337 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/ba929c69259341e3839018159ddd7b89 2024-11-21T21:25:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/0fe431059e7145dea4997abca1425f3b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/0fe431059e7145dea4997abca1425f3b 2024-11-21T21:25:36,356 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/0fe431059e7145dea4997abca1425f3b, entries=150, sequenceid=192, filesize=30.4 K 2024-11-21T21:25:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/3f93f033ac2c4625b44ccf1fcfbb90b3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3f93f033ac2c4625b44ccf1fcfbb90b3 2024-11-21T21:25:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,367 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3f93f033ac2c4625b44ccf1fcfbb90b3, entries=150, sequenceid=192, filesize=11.9 K 2024-11-21T21:25:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/ba929c69259341e3839018159ddd7b89 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/ba929c69259341e3839018159ddd7b89 2024-11-21T21:25:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,375 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/ba929c69259341e3839018159ddd7b89, entries=150, sequenceid=192, filesize=11.9 K 2024-11-21T21:25:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,377 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 58ad0d609f8c768bc6016e57e5d831ac in 557ms, sequenceid=192, compaction requested=false 2024-11-21T21:25:36,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:36,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:36,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-21T21:25:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-21T21:25:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T21:25:36,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6420 sec 2024-11-21T21:25:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.6480 sec 2024-11-21T21:25:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:36,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:25:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218919caceefba41e0b4f3d05855a614a1_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224336837/Put/seqid=0 2024-11-21T21:25:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742243_1419 (size=24758) 2024-11-21T21:25:36,873 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:36,877 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218919caceefba41e0b4f3d05855a614a1_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218919caceefba41e0b4f3d05855a614a1_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:36,878 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/b6937195b6bd44d58f3796facb16e9db, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:36,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/b6937195b6bd44d58f3796facb16e9db is 175, key is test_row_0/A:col10/1732224336837/Put/seqid=0 2024-11-21T21:25:36,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742244_1420 (size=74395) 2024-11-21T21:25:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:25:36,890 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:36,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-21T21:25:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:25:36,893 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:36,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:36,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:36,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224396897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:36,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224396898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:36,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224396899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:36,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224396900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224396901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:25:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224397020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224397020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224397020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224397021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224397021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T21:25:37,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:37,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T21:25:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:25:37,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224397229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224397242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224397242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224397242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224397242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,282 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/b6937195b6bd44d58f3796facb16e9db 2024-11-21T21:25:37,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/7d5ebd7c532d4ad4a731070578883b5c is 50, key is test_row_0/B:col10/1732224336837/Put/seqid=0 2024-11-21T21:25:37,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742245_1421 (size=12151) 2024-11-21T21:25:37,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/7d5ebd7c532d4ad4a731070578883b5c 2024-11-21T21:25:37,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/eed49dc187754621aea9e916a21ddedd is 50, key is test_row_0/C:col10/1732224336837/Put/seqid=0 2024-11-21T21:25:37,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T21:25:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742246_1422 (size=12151) 2024-11-21T21:25:37,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T21:25:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:25:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224397550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224397550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224397550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224397550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224397550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:37,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T21:25:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:37,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/eed49dc187754621aea9e916a21ddedd 2024-11-21T21:25:37,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/b6937195b6bd44d58f3796facb16e9db as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/b6937195b6bd44d58f3796facb16e9db 2024-11-21T21:25:37,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/b6937195b6bd44d58f3796facb16e9db, entries=400, sequenceid=206, filesize=72.7 K 2024-11-21T21:25:37,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/7d5ebd7c532d4ad4a731070578883b5c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/7d5ebd7c532d4ad4a731070578883b5c 2024-11-21T21:25:37,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/7d5ebd7c532d4ad4a731070578883b5c, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T21:25:37,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/eed49dc187754621aea9e916a21ddedd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/eed49dc187754621aea9e916a21ddedd 2024-11-21T21:25:37,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/eed49dc187754621aea9e916a21ddedd, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T21:25:37,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 58ad0d609f8c768bc6016e57e5d831ac in 951ms, sequenceid=206, compaction requested=true 2024-11-21T21:25:37,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:37,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:37,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:37,789 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:37,789 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:37,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:37,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:37,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:37,790 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:37,790 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:37,790 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/B is initiating minor compaction (all files) 2024-11-21T21:25:37,790 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/B in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,791 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/84bb5c8d5ac1465f880c7cb8bcf7bc27, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3f93f033ac2c4625b44ccf1fcfbb90b3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/7d5ebd7c532d4ad4a731070578883b5c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=36.0 K 2024-11-21T21:25:37,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:37,791 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/A is initiating minor compaction (all files) 2024-11-21T21:25:37,791 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/A in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,791 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 84bb5c8d5ac1465f880c7cb8bcf7bc27, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732224334255 2024-11-21T21:25:37,791 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/2e776e3cb2114547b7bff924402d98b4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/0fe431059e7145dea4997abca1425f3b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/b6937195b6bd44d58f3796facb16e9db] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=133.8 K 2024-11-21T21:25:37,791 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,791 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/2e776e3cb2114547b7bff924402d98b4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/0fe431059e7145dea4997abca1425f3b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/b6937195b6bd44d58f3796facb16e9db] 2024-11-21T21:25:37,791 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f93f033ac2c4625b44ccf1fcfbb90b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732224334533 2024-11-21T21:25:37,791 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d5ebd7c532d4ad4a731070578883b5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224336801 2024-11-21T21:25:37,791 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e776e3cb2114547b7bff924402d98b4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732224334255 2024-11-21T21:25:37,792 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe431059e7145dea4997abca1425f3b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732224334533 2024-11-21T21:25:37,792 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6937195b6bd44d58f3796facb16e9db, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224336792 2024-11-21T21:25:37,814 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:37,814 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/6115e850f548411b9d7e822fac7a46f6 is 50, key is test_row_0/B:col10/1732224336837/Put/seqid=0 2024-11-21T21:25:37,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T21:25:37,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,816 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:25:37,821 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:37,866 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112140d35096ccd44b0cab784ba5555dfeb8_58ad0d609f8c768bc6016e57e5d831ac store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:37,870 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112140d35096ccd44b0cab784ba5555dfeb8_58ad0d609f8c768bc6016e57e5d831ac, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:37,870 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112140d35096ccd44b0cab784ba5555dfeb8_58ad0d609f8c768bc6016e57e5d831ac because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742247_1423 (size=12629) 2024-11-21T21:25:37,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215c5d70cfce5742a08b283937795ac1bc_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224336861/Put/seqid=0 2024-11-21T21:25:37,882 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/6115e850f548411b9d7e822fac7a46f6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/6115e850f548411b9d7e822fac7a46f6 2024-11-21T21:25:37,892 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/B of 58ad0d609f8c768bc6016e57e5d831ac into 6115e850f548411b9d7e822fac7a46f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:37,892 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:37,892 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/B, priority=13, startTime=1732224337789; duration=0sec 2024-11-21T21:25:37,892 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:37,893 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:B 2024-11-21T21:25:37,893 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:37,894 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:37,894 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/C is initiating minor compaction (all files) 2024-11-21T21:25:37,894 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/C in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:37,894 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/463c8da8e0fb4243853329ff6a42b97d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/ba929c69259341e3839018159ddd7b89, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/eed49dc187754621aea9e916a21ddedd] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=36.0 K 2024-11-21T21:25:37,897 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 463c8da8e0fb4243853329ff6a42b97d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732224334255 2024-11-21T21:25:37,897 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ba929c69259341e3839018159ddd7b89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732224334533 2024-11-21T21:25:37,897 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting eed49dc187754621aea9e916a21ddedd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224336801 2024-11-21T21:25:37,923 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:37,925 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/d84128a3ea2247deb98828f5fadde1da is 50, key is test_row_0/C:col10/1732224336837/Put/seqid=0 2024-11-21T21:25:37,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742248_1424 (size=4469) 2024-11-21T21:25:37,931 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#A#compaction#355 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:37,932 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/f7100fd22fb943adb150a33deb05f089 is 175, key is test_row_0/A:col10/1732224336837/Put/seqid=0 2024-11-21T21:25:37,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742249_1425 (size=12304) 2024-11-21T21:25:37,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742251_1427 (size=31583) 2024-11-21T21:25:37,992 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/f7100fd22fb943adb150a33deb05f089 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f7100fd22fb943adb150a33deb05f089 2024-11-21T21:25:37,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742250_1426 (size=12629) 2024-11-21T21:25:37,997 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/A of 58ad0d609f8c768bc6016e57e5d831ac into f7100fd22fb943adb150a33deb05f089(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:37,997 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:37,997 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/A, priority=13, startTime=1732224337789; duration=0sec 2024-11-21T21:25:37,997 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:37,997 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:A 2024-11-21T21:25:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:25:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:38,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:38,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224398071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224398074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224398076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224398080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224398081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224398190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224398190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224398200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224398200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224398204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:38,354 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215c5d70cfce5742a08b283937795ac1bc_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215c5d70cfce5742a08b283937795ac1bc_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/ccbc75b3ad694d81b389644999c040cd, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/ccbc75b3ad694d81b389644999c040cd is 175, key is test_row_0/A:col10/1732224336861/Put/seqid=0 2024-11-21T21:25:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742252_1428 (size=31105) 2024-11-21T21:25:38,366 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/ccbc75b3ad694d81b389644999c040cd 2024-11-21T21:25:38,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:25:38,370 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T21:25:38,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/d576a0b12909498388d391a1fb108f30 is 50, key is test_row_0/B:col10/1732224336861/Put/seqid=0 2024-11-21T21:25:38,403 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/d84128a3ea2247deb98828f5fadde1da as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/d84128a3ea2247deb98828f5fadde1da 2024-11-21T21:25:38,409 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/C of 58ad0d609f8c768bc6016e57e5d831ac into d84128a3ea2247deb98828f5fadde1da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:38,409 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:38,409 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/C, priority=13, startTime=1732224337790; duration=0sec 2024-11-21T21:25:38,409 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:38,409 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:C 2024-11-21T21:25:38,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224398410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224398410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224398419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224398419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224398419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742253_1429 (size=12151) 2024-11-21T21:25:38,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224398730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224398730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224398739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224398739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224398741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:38,831 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/d576a0b12909498388d391a1fb108f30 2024-11-21T21:25:38,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/e9b9f12b6def4cff80a7dcfe8ddcfe05 is 50, key is test_row_0/C:col10/1732224336861/Put/seqid=0 2024-11-21T21:25:38,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742254_1430 (size=12151) 2024-11-21T21:25:38,873 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/e9b9f12b6def4cff80a7dcfe8ddcfe05 2024-11-21T21:25:38,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/ccbc75b3ad694d81b389644999c040cd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ccbc75b3ad694d81b389644999c040cd 2024-11-21T21:25:38,891 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ccbc75b3ad694d81b389644999c040cd, entries=150, sequenceid=231, filesize=30.4 K 2024-11-21T21:25:38,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/d576a0b12909498388d391a1fb108f30 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d576a0b12909498388d391a1fb108f30 2024-11-21T21:25:38,896 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d576a0b12909498388d391a1fb108f30, entries=150, sequenceid=231, filesize=11.9 K 2024-11-21T21:25:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/e9b9f12b6def4cff80a7dcfe8ddcfe05 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e9b9f12b6def4cff80a7dcfe8ddcfe05 2024-11-21T21:25:38,904 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e9b9f12b6def4cff80a7dcfe8ddcfe05, entries=150, sequenceid=231, filesize=11.9 K 2024-11-21T21:25:38,910 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 58ad0d609f8c768bc6016e57e5d831ac in 1094ms, sequenceid=231, compaction requested=false 2024-11-21T21:25:38,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:38,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-21T21:25:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-21T21:25:38,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T21:25:38,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-11-21T21:25:38,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0320 sec 2024-11-21T21:25:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:25:39,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:39,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-21T21:25:39,053 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:39,054 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:39,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:25:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:25:39,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T21:25:39,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:39,207 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:25:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ce2c5ca94af240edaffb495e37a880df_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224338074/Put/seqid=0 2024-11-21T21:25:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742255_1431 (size=12304) 2024-11-21T21:25:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:39,247 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ce2c5ca94af240edaffb495e37a880df_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ce2c5ca94af240edaffb495e37a880df_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:39,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:39,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e48d330bfb76482a95c3ef3c0bd760cb, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:39,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e48d330bfb76482a95c3ef3c0bd760cb is 175, key is test_row_0/A:col10/1732224338074/Put/seqid=0 2024-11-21T21:25:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742256_1432 (size=31105) 2024-11-21T21:25:39,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224399296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224399297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224399298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224399298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224399321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:25:39,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224399420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224399421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224399421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224399421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224399433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224399630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224399640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224399640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224399640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224399650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:25:39,682 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e48d330bfb76482a95c3ef3c0bd760cb 2024-11-21T21:25:39,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9 is 50, key is test_row_0/B:col10/1732224338074/Put/seqid=0 2024-11-21T21:25:39,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742257_1433 (size=12151) 2024-11-21T21:25:39,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224399950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224399959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224399959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224399959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224399960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:40,091 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9 2024-11-21T21:25:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/e5d56fc506f84cd1b181bfd157dcfef1 is 50, key is test_row_0/C:col10/1732224338074/Put/seqid=0 2024-11-21T21:25:40,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742258_1434 (size=12151) 2024-11-21T21:25:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:25:40,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224400470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224400470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224400470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224400470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224400470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:40,520 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/e5d56fc506f84cd1b181bfd157dcfef1 2024-11-21T21:25:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e48d330bfb76482a95c3ef3c0bd760cb as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e48d330bfb76482a95c3ef3c0bd760cb 2024-11-21T21:25:40,529 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e48d330bfb76482a95c3ef3c0bd760cb, entries=150, sequenceid=246, filesize=30.4 K 2024-11-21T21:25:40,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9 2024-11-21T21:25:40,537 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9, entries=150, sequenceid=246, filesize=11.9 K 2024-11-21T21:25:40,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/e5d56fc506f84cd1b181bfd157dcfef1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e5d56fc506f84cd1b181bfd157dcfef1 2024-11-21T21:25:40,543 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e5d56fc506f84cd1b181bfd157dcfef1, entries=150, sequenceid=246, filesize=11.9 K 2024-11-21T21:25:40,544 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 58ad0d609f8c768bc6016e57e5d831ac in 1337ms, sequenceid=246, compaction requested=true 2024-11-21T21:25:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-21T21:25:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-21T21:25:40,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T21:25:40,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-11-21T21:25:40,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5030 sec 2024-11-21T21:25:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:25:41,200 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:41,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-21T21:25:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:25:41,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:41,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:41,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:25:41,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:25:41,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:41,355 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:25:41,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b65f43935f6c48dfb93147e8b9b71bb0_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224339297/Put/seqid=0 2024-11-21T21:25:41,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742259_1435 (size=12454) 2024-11-21T21:25:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:41,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224401495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224401501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224401502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224401510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224401510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:25:41,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224401619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224401619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224401620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224401620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224401620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:41,780 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b65f43935f6c48dfb93147e8b9b71bb0_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b65f43935f6c48dfb93147e8b9b71bb0_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/7665f9b9d5734c4d8180fd8f438143a0, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/7665f9b9d5734c4d8180fd8f438143a0 is 175, key is test_row_0/A:col10/1732224339297/Put/seqid=0 2024-11-21T21:25:41,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742260_1436 (size=31255) 2024-11-21T21:25:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:25:41,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224401829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224402139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224402160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224402160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224402160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224402160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,184 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/7665f9b9d5734c4d8180fd8f438143a0 2024-11-21T21:25:42,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/e349e83ed7574e4c830b8e4ae73bff14 is 50, key is test_row_0/B:col10/1732224339297/Put/seqid=0 2024-11-21T21:25:42,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742261_1437 (size=12301) 2024-11-21T21:25:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:25:42,594 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/e349e83ed7574e4c830b8e4ae73bff14 2024-11-21T21:25:42,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/5983f318387e411fb73006fbafbc2923 is 50, key is test_row_0/C:col10/1732224339297/Put/seqid=0 2024-11-21T21:25:42,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742262_1438 (size=12301) 2024-11-21T21:25:42,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224402659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224402670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224402670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224402679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224402679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,003 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/5983f318387e411fb73006fbafbc2923 2024-11-21T21:25:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/7665f9b9d5734c4d8180fd8f438143a0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/7665f9b9d5734c4d8180fd8f438143a0 2024-11-21T21:25:43,009 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/7665f9b9d5734c4d8180fd8f438143a0, entries=150, sequenceid=270, filesize=30.5 K 2024-11-21T21:25:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/e349e83ed7574e4c830b8e4ae73bff14 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e349e83ed7574e4c830b8e4ae73bff14 2024-11-21T21:25:43,013 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e349e83ed7574e4c830b8e4ae73bff14, entries=150, sequenceid=270, filesize=12.0 K 2024-11-21T21:25:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/5983f318387e411fb73006fbafbc2923 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/5983f318387e411fb73006fbafbc2923 2024-11-21T21:25:43,016 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/5983f318387e411fb73006fbafbc2923, entries=150, sequenceid=270, filesize=12.0 K 2024-11-21T21:25:43,017 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 58ad0d609f8c768bc6016e57e5d831ac in 1662ms, sequenceid=270, compaction requested=true 2024-11-21T21:25:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-21T21:25:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-21T21:25:43,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T21:25:43,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-11-21T21:25:43,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8200 sec 2024-11-21T21:25:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:25:43,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:43,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-21T21:25:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:25:43,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:43,352 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:43,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:25:43,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T21:25:43,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:43,504 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:25:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121295756079dc74f28aa688c6871f6a350_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224341508/Put/seqid=0 2024-11-21T21:25:43,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742263_1439 (size=12454) 2024-11-21T21:25:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:25:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:43,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224403715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224403721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224403721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224403722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224403725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224403829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224403829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224403829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224403840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224403840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:43,916 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121295756079dc74f28aa688c6871f6a350_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121295756079dc74f28aa688c6871f6a350_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:43,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44e0a61fed2545c0b80651d588336a5e, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:43,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44e0a61fed2545c0b80651d588336a5e is 175, key is test_row_0/A:col10/1732224341508/Put/seqid=0 2024-11-21T21:25:43,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742264_1440 (size=31255) 2024-11-21T21:25:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:25:44,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224404039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224404040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224404040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224404059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224404059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,320 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44e0a61fed2545c0b80651d588336a5e 2024-11-21T21:25:44,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/efa3bfaaa2824799a30c5bdfaf43d57c is 50, key is test_row_0/B:col10/1732224341508/Put/seqid=0 2024-11-21T21:25:44,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742265_1441 (size=12301) 2024-11-21T21:25:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224404359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224404360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224404360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224404370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224404370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:25:44,729 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/efa3bfaaa2824799a30c5bdfaf43d57c 2024-11-21T21:25:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/486b380136dc47d5961d5291c9eab69f is 50, key is test_row_0/C:col10/1732224341508/Put/seqid=0 2024-11-21T21:25:44,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742266_1442 (size=12301) 2024-11-21T21:25:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224404870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224404880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224404880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224404889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224404889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:45,137 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/486b380136dc47d5961d5291c9eab69f 2024-11-21T21:25:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/44e0a61fed2545c0b80651d588336a5e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44e0a61fed2545c0b80651d588336a5e 2024-11-21T21:25:45,143 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44e0a61fed2545c0b80651d588336a5e, entries=150, sequenceid=283, filesize=30.5 K 2024-11-21T21:25:45,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/efa3bfaaa2824799a30c5bdfaf43d57c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/efa3bfaaa2824799a30c5bdfaf43d57c 2024-11-21T21:25:45,146 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/efa3bfaaa2824799a30c5bdfaf43d57c, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T21:25:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/486b380136dc47d5961d5291c9eab69f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/486b380136dc47d5961d5291c9eab69f 2024-11-21T21:25:45,150 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/486b380136dc47d5961d5291c9eab69f, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T21:25:45,150 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 58ad0d609f8c768bc6016e57e5d831ac in 1647ms, sequenceid=283, compaction requested=true 2024-11-21T21:25:45,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:45,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:45,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-21T21:25:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-21T21:25:45,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T21:25:45,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-21T21:25:45,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8030 sec 2024-11-21T21:25:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:25:45,500 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:45,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-21T21:25:45,502 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:25:45,503 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:45,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:25:45,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T21:25:45,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:45,655 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:25:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c661ff3561b44ae19804f510b49a1f90_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224343724/Put/seqid=0 2024-11-21T21:25:45,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742267_1443 (size=12454) 2024-11-21T21:25:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:45,669 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c661ff3561b44ae19804f510b49a1f90_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c661ff3561b44ae19804f510b49a1f90_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:45,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e9c0bd89d6e449afadaa2d59d985d88b, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:45,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e9c0bd89d6e449afadaa2d59d985d88b is 175, key is test_row_0/A:col10/1732224343724/Put/seqid=0 2024-11-21T21:25:45,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742268_1444 (size=31255) 2024-11-21T21:25:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:25:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:45,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:45,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224405919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:45,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224405933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:45,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224405962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224405962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:45,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224405962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,075 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e9c0bd89d6e449afadaa2d59d985d88b 2024-11-21T21:25:46,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224406070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224406070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/4eabf5887aab43e49eaa3001cf7fd0b7 is 50, key is test_row_0/B:col10/1732224343724/Put/seqid=0 2024-11-21T21:25:46,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742269_1445 (size=12301) 2024-11-21T21:25:46,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224406080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224406080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224406080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:25:46,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224406290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224406290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224406299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224406299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224406299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,485 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/4eabf5887aab43e49eaa3001cf7fd0b7 2024-11-21T21:25:46,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/28878e1de9344456bb5772a019894423 is 50, key is test_row_0/C:col10/1732224343724/Put/seqid=0 2024-11-21T21:25:46,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742270_1446 (size=12301) 2024-11-21T21:25:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224406600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224406600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224406619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224406619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224406619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:25:46,900 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/28878e1de9344456bb5772a019894423 2024-11-21T21:25:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e9c0bd89d6e449afadaa2d59d985d88b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e9c0bd89d6e449afadaa2d59d985d88b 2024-11-21T21:25:46,907 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e9c0bd89d6e449afadaa2d59d985d88b, entries=150, sequenceid=306, filesize=30.5 K 2024-11-21T21:25:46,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/4eabf5887aab43e49eaa3001cf7fd0b7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/4eabf5887aab43e49eaa3001cf7fd0b7 2024-11-21T21:25:46,910 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/4eabf5887aab43e49eaa3001cf7fd0b7, entries=150, sequenceid=306, filesize=12.0 K 2024-11-21T21:25:46,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/28878e1de9344456bb5772a019894423 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/28878e1de9344456bb5772a019894423 2024-11-21T21:25:46,914 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/28878e1de9344456bb5772a019894423, entries=150, sequenceid=306, filesize=12.0 K 2024-11-21T21:25:46,914 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 58ad0d609f8c768bc6016e57e5d831ac in 1259ms, sequenceid=306, compaction requested=true 2024-11-21T21:25:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:46,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-21T21:25:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-21T21:25:46,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T21:25:46,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-11-21T21:25:46,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.4170 sec 2024-11-21T21:25:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:47,124 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:47,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:25:47,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121687c944fc50f47eda2f27427ee91cf38_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224345932/Put/seqid=0 2024-11-21T21:25:47,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742271_1447 (size=17534) 2024-11-21T21:25:47,139 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:25:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224407165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224407172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224407174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224407174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224407175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224407279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224407279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224407289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224407299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224407299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224407499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224407499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224407509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224407510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224407520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,533 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:47,536 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121687c944fc50f47eda2f27427ee91cf38_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121687c944fc50f47eda2f27427ee91cf38_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:47,537 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e94e5ad96ff8492fb32a8ce4c7e78196, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:47,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e94e5ad96ff8492fb32a8ce4c7e78196 is 175, key is test_row_0/A:col10/1732224345932/Put/seqid=0 2024-11-21T21:25:47,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742272_1448 (size=48639) 2024-11-21T21:25:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:25:47,650 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:47,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-21T21:25:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:25:47,652 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:47,652 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:47,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:25:47,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:25:47,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:47,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224407819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224407819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224407819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224407819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224407830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:47,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e94e5ad96ff8492fb32a8ce4c7e78196 2024-11-21T21:25:47,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/98a174057e704ad88d9200bab7cb6a47 is 50, key is test_row_0/B:col10/1732224345932/Put/seqid=0 2024-11-21T21:25:47,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:25:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:47,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742273_1449 (size=12301) 2024-11-21T21:25:47,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/98a174057e704ad88d9200bab7cb6a47 2024-11-21T21:25:47,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/c6663e59f2124131949594ade4f18d9c is 50, key is test_row_0/C:col10/1732224345932/Put/seqid=0 2024-11-21T21:25:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:25:47,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742274_1450 (size=12301) 2024-11-21T21:25:48,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:25:48,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:48,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:48,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:48,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:25:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:25:48,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1732224408340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:48,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34994 deadline: 1732224408340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:48,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732224408340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:48,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34984 deadline: 1732224408340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:48,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35004 deadline: 1732224408349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:25:48,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/c6663e59f2124131949594ade4f18d9c 2024-11-21T21:25:48,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/e94e5ad96ff8492fb32a8ce4c7e78196 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e94e5ad96ff8492fb32a8ce4c7e78196 2024-11-21T21:25:48,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e94e5ad96ff8492fb32a8ce4c7e78196, entries=250, sequenceid=321, filesize=47.5 K 2024-11-21T21:25:48,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/98a174057e704ad88d9200bab7cb6a47 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/98a174057e704ad88d9200bab7cb6a47 2024-11-21T21:25:48,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/98a174057e704ad88d9200bab7cb6a47, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T21:25:48,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/c6663e59f2124131949594ade4f18d9c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c6663e59f2124131949594ade4f18d9c 2024-11-21T21:25:48,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c6663e59f2124131949594ade4f18d9c, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T21:25:48,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 58ad0d609f8c768bc6016e57e5d831ac in 1261ms, sequenceid=321, compaction requested=true 2024-11-21T21:25:48,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:48,386 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-21T21:25:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:48,386 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-21T21:25:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58ad0d609f8c768bc6016e57e5d831ac:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:48,387 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 236197 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-21T21:25:48,387 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/A is initiating minor compaction (all files) 2024-11-21T21:25:48,387 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/A in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,387 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f7100fd22fb943adb150a33deb05f089, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ccbc75b3ad694d81b389644999c040cd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e48d330bfb76482a95c3ef3c0bd760cb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/7665f9b9d5734c4d8180fd8f438143a0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44e0a61fed2545c0b80651d588336a5e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e9c0bd89d6e449afadaa2d59d985d88b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e94e5ad96ff8492fb32a8ce4c7e78196] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=230.7 K 2024-11-21T21:25:48,387 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,387 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f7100fd22fb943adb150a33deb05f089, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ccbc75b3ad694d81b389644999c040cd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e48d330bfb76482a95c3ef3c0bd760cb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/7665f9b9d5734c4d8180fd8f438143a0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44e0a61fed2545c0b80651d588336a5e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e9c0bd89d6e449afadaa2d59d985d88b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e94e5ad96ff8492fb32a8ce4c7e78196] 2024-11-21T21:25:48,387 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86135 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-21T21:25:48,387 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7100fd22fb943adb150a33deb05f089, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224336801 2024-11-21T21:25:48,387 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/B is initiating minor compaction (all files) 2024-11-21T21:25:48,387 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/B in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,388 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/6115e850f548411b9d7e822fac7a46f6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d576a0b12909498388d391a1fb108f30, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e349e83ed7574e4c830b8e4ae73bff14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/efa3bfaaa2824799a30c5bdfaf43d57c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/4eabf5887aab43e49eaa3001cf7fd0b7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/98a174057e704ad88d9200bab7cb6a47] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=84.1 K 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccbc75b3ad694d81b389644999c040cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732224336861 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e48d330bfb76482a95c3ef3c0bd760cb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732224338071 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6115e850f548411b9d7e822fac7a46f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224336801 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7665f9b9d5734c4d8180fd8f438143a0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732224339297 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting d576a0b12909498388d391a1fb108f30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732224336861 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e0a61fed2545c0b80651d588336a5e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732224341500 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c0f0b75ab24b0fb26ea6e462ff3ac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732224338071 2024-11-21T21:25:48,388 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9c0bd89d6e449afadaa2d59d985d88b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732224343719 2024-11-21T21:25:48,389 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e349e83ed7574e4c830b8e4ae73bff14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732224339297 2024-11-21T21:25:48,389 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e94e5ad96ff8492fb32a8ce4c7e78196, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732224345932 2024-11-21T21:25:48,389 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting efa3bfaaa2824799a30c5bdfaf43d57c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732224341500 2024-11-21T21:25:48,389 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eabf5887aab43e49eaa3001cf7fd0b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732224343719 2024-11-21T21:25:48,389 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a174057e704ad88d9200bab7cb6a47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732224345932 2024-11-21T21:25:48,396 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:48,397 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112150c5fc1c1f824a468fb0adf57663806c_58ad0d609f8c768bc6016e57e5d831ac store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:48,400 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112150c5fc1c1f824a468fb0adf57663806c_58ad0d609f8c768bc6016e57e5d831ac, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:48,400 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112150c5fc1c1f824a468fb0adf57663806c_58ad0d609f8c768bc6016e57e5d831ac because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:48,406 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#B#compaction#376 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:48,406 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/88b51d4959b8412989e7df189c1176cf is 50, key is test_row_0/B:col10/1732224345932/Put/seqid=0 2024-11-21T21:25:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742275_1451 (size=4469) 2024-11-21T21:25:48,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742276_1452 (size=13017) 2024-11-21T21:25:48,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:25:48,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,415 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:25:48,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112178b0aac648514e5ab33a8cc9352d95cd_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224347174/Put/seqid=0 2024-11-21T21:25:48,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742277_1453 (size=12454) 2024-11-21T21:25:48,662 INFO [Thread-1667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1667. 2024-11-21T21:25:48,662 DEBUG [Thread-1667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:48,662 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,662 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:48,663 INFO [Thread-1673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1673. 2024-11-21T21:25:48,663 DEBUG [Thread-1673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:48,663 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,663 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:48,664 INFO [Thread-1665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1665. 2024-11-21T21:25:48,664 DEBUG [Thread-1665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:48,664 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,665 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:48,671 INFO [Thread-1669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1669. 2024-11-21T21:25:48,671 DEBUG [Thread-1669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:48,671 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,671 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:48,673 INFO [Thread-1671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1671. 2024-11-21T21:25:48,673 DEBUG [Thread-1671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:48,673 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,673 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:48,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:25:48,810 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#A#compaction#375 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:48,811 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/f4f2639dfbff4ff1a2fb00f5ad98c776 is 175, key is test_row_0/A:col10/1732224345932/Put/seqid=0 2024-11-21T21:25:48,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742278_1454 (size=31971) 2024-11-21T21:25:48,821 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/88b51d4959b8412989e7df189c1176cf as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/88b51d4959b8412989e7df189c1176cf 2024-11-21T21:25:48,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:48,826 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/B of 58ad0d609f8c768bc6016e57e5d831ac into 88b51d4959b8412989e7df189c1176cf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:48,826 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:48,826 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/B, priority=9, startTime=1732224348386; duration=0sec 2024-11-21T21:25:48,827 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:48,827 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:B 2024-11-21T21:25:48,827 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-21T21:25:48,828 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112178b0aac648514e5ab33a8cc9352d95cd_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112178b0aac648514e5ab33a8cc9352d95cd_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:48,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/c7d6ea0ccb444a9799230d976a05ab1d, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:48,828 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86135 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-21T21:25:48,829 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 58ad0d609f8c768bc6016e57e5d831ac/C is initiating minor compaction (all files) 2024-11-21T21:25:48,829 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58ad0d609f8c768bc6016e57e5d831ac/C in TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:48,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/c7d6ea0ccb444a9799230d976a05ab1d is 175, key is test_row_0/A:col10/1732224347174/Put/seqid=0 2024-11-21T21:25:48,829 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/d84128a3ea2247deb98828f5fadde1da, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e9b9f12b6def4cff80a7dcfe8ddcfe05, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e5d56fc506f84cd1b181bfd157dcfef1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/5983f318387e411fb73006fbafbc2923, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/486b380136dc47d5961d5291c9eab69f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/28878e1de9344456bb5772a019894423, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c6663e59f2124131949594ade4f18d9c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp, totalSize=84.1 K 2024-11-21T21:25:48,829 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting d84128a3ea2247deb98828f5fadde1da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224336801 2024-11-21T21:25:48,829 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b9f12b6def4cff80a7dcfe8ddcfe05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732224336861 2024-11-21T21:25:48,830 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d56fc506f84cd1b181bfd157dcfef1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732224338071 2024-11-21T21:25:48,830 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5983f318387e411fb73006fbafbc2923, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732224339297 2024-11-21T21:25:48,830 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 486b380136dc47d5961d5291c9eab69f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732224341500 2024-11-21T21:25:48,830 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 28878e1de9344456bb5772a019894423, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732224343719 2024-11-21T21:25:48,831 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c6663e59f2124131949594ade4f18d9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732224345932 2024-11-21T21:25:48,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742279_1455 (size=31255) 2024-11-21T21:25:48,843 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58ad0d609f8c768bc6016e57e5d831ac#C#compaction#378 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:48,844 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/9414db46a8094d2e8e9d3f38b8cb9c7a is 50, key is test_row_0/C:col10/1732224345932/Put/seqid=0 2024-11-21T21:25:48,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742280_1456 (size=13017) 2024-11-21T21:25:49,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/f4f2639dfbff4ff1a2fb00f5ad98c776 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f4f2639dfbff4ff1a2fb00f5ad98c776 2024-11-21T21:25:49,233 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/c7d6ea0ccb444a9799230d976a05ab1d 2024-11-21T21:25:49,237 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/A of 58ad0d609f8c768bc6016e57e5d831ac into f4f2639dfbff4ff1a2fb00f5ad98c776(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:49,237 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:49,237 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/A, priority=9, startTime=1732224348385; duration=0sec 2024-11-21T21:25:49,237 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:49,237 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:A 2024-11-21T21:25:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/62a82309622b42488b605d8ae6afeaa4 is 50, key is test_row_0/B:col10/1732224347174/Put/seqid=0 2024-11-21T21:25:49,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742281_1457 (size=12301) 2024-11-21T21:25:49,251 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/9414db46a8094d2e8e9d3f38b8cb9c7a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9414db46a8094d2e8e9d3f38b8cb9c7a 2024-11-21T21:25:49,255 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 58ad0d609f8c768bc6016e57e5d831ac/C of 58ad0d609f8c768bc6016e57e5d831ac into 9414db46a8094d2e8e9d3f38b8cb9c7a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:49,255 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:49,255 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., storeName=58ad0d609f8c768bc6016e57e5d831ac/C, priority=9, startTime=1732224348386; duration=0sec 2024-11-21T21:25:49,255 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:49,256 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58ad0d609f8c768bc6016e57e5d831ac:C 2024-11-21T21:25:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:49,366 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-11-21T21:25:49,366 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-11-21T21:25:49,366 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:49,366 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:49,366 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,366 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,366 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,366 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:49,367 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-11-21T21:25:49,367 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:49,367 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:49,367 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. as already flushing 2024-11-21T21:25:49,369 INFO [Thread-1660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1660. 2024-11-21T21:25:49,369 DEBUG [Thread-1660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:49,369 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,369 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:49,372 INFO [Thread-1662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1662. 2024-11-21T21:25:49,372 DEBUG [Thread-1662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:25:49,372 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,372 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:49,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:49,646 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/62a82309622b42488b605d8ae6afeaa4 2024-11-21T21:25:49,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/41b10208889e4ea18fa7b13b92fd0863 is 50, key is test_row_0/C:col10/1732224347174/Put/seqid=0 2024-11-21T21:25:49,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742282_1458 (size=12301) 2024-11-21T21:25:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:25:50,065 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/41b10208889e4ea18fa7b13b92fd0863 2024-11-21T21:25:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/c7d6ea0ccb444a9799230d976a05ab1d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/c7d6ea0ccb444a9799230d976a05ab1d 2024-11-21T21:25:50,081 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/c7d6ea0ccb444a9799230d976a05ab1d, entries=150, sequenceid=342, filesize=30.5 K 2024-11-21T21:25:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/62a82309622b42488b605d8ae6afeaa4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/62a82309622b42488b605d8ae6afeaa4 2024-11-21T21:25:50,087 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/62a82309622b42488b605d8ae6afeaa4, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T21:25:50,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/41b10208889e4ea18fa7b13b92fd0863 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/41b10208889e4ea18fa7b13b92fd0863 2024-11-21T21:25:50,093 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/41b10208889e4ea18fa7b13b92fd0863, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T21:25:50,094 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=33.54 KB/34350 for 58ad0d609f8c768bc6016e57e5d831ac in 1679ms, sequenceid=342, compaction requested=false 2024-11-21T21:25:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 58ad0d609f8c768bc6016e57e5d831ac: 2024-11-21T21:25:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-21T21:25:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-21T21:25:50,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T21:25:50,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4420 sec 2024-11-21T21:25:50,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.4480 sec 2024-11-21T21:25:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:25:51,820 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:51,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:25:51,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-21T21:25:51,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-21T21:25:51,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-21T21:25:51,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1056 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3142 rows 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1043 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3129 rows 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3173 rows 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1060 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3145 rows 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1050 2024-11-21T21:25:51,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3150 rows 2024-11-21T21:25:51,821 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:25:51,821 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:25:51,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:51,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:51,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:25:51,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:25:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:25:51,827 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224351827"}]},"ts":"1732224351827"} 2024-11-21T21:25:51,829 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:25:51,829 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:25:51,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:25:51,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, UNASSIGN}] 2024-11-21T21:25:51,833 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, UNASSIGN 2024-11-21T21:25:51,834 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:51,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, UNASSIGN because future has completed 2024-11-21T21:25:51,835 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:25:51,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:25:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:25:51,989 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:51,989 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:25:51,989 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 58ad0d609f8c768bc6016e57e5d831ac, disabling compactions & flushes 2024-11-21T21:25:51,989 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:51,989 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:51,989 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. after waiting 0 ms 2024-11-21T21:25:51,989 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:51,990 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 58ad0d609f8c768bc6016e57e5d831ac 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T21:25:52,000 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c59447db78274d0fab6dc74a0e953b40_58ad0d609f8c768bc6016e57e5d831ac is 50, key is test_row_0/A:col10/1732224349370/Put/seqid=0 2024-11-21T21:25:52,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742283_1459 (size=12454) 2024-11-21T21:25:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:25:52,403 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:52,406 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c59447db78274d0fab6dc74a0e953b40_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c59447db78274d0fab6dc74a0e953b40_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:52,407 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/9df90ab8e4204973a41d753f8684dbb7, store: [table=TestAcidGuarantees family=A region=58ad0d609f8c768bc6016e57e5d831ac] 2024-11-21T21:25:52,407 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/9df90ab8e4204973a41d753f8684dbb7 is 175, key is test_row_0/A:col10/1732224349370/Put/seqid=0 2024-11-21T21:25:52,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742284_1460 (size=31255) 2024-11-21T21:25:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:25:52,525 WARN [regionserver/849dee24a242:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-21T21:25:52,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44bfe8b3a6f046beb8bd50499618e65c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/42e460b5ea0e48e08d29ffe25a8c8cc8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/637ccf8b96a243e7949b7f1bc924b15e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/77cf3db2b9c04cfb94538f379fa4b71c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/866f8358cf1443339c3a56f4c7e22a08, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/277d89bdcef243fe99857b5b2f9140a2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/18d6af7d5ae5471582bd4cc159d0c3ae, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/dab4762355214ab7ad505d1c2bb308a9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ee8ad6944b6e48a68a6af892ee73ebf0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/3323368efd8b475f8085edeccd1765bc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/d03986302a2e4b41ba22ed329232b5ad, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/2e776e3cb2114547b7bff924402d98b4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/0fe431059e7145dea4997abca1425f3b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/b6937195b6bd44d58f3796facb16e9db, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f7100fd22fb943adb150a33deb05f089, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ccbc75b3ad694d81b389644999c040cd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e48d330bfb76482a95c3ef3c0bd760cb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/7665f9b9d5734c4d8180fd8f438143a0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44e0a61fed2545c0b80651d588336a5e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e9c0bd89d6e449afadaa2d59d985d88b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e94e5ad96ff8492fb32a8ce4c7e78196] to archive 2024-11-21T21:25:52,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:25:52,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44bfe8b3a6f046beb8bd50499618e65c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44bfe8b3a6f046beb8bd50499618e65c 2024-11-21T21:25:52,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/42e460b5ea0e48e08d29ffe25a8c8cc8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/42e460b5ea0e48e08d29ffe25a8c8cc8 2024-11-21T21:25:52,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/637ccf8b96a243e7949b7f1bc924b15e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/637ccf8b96a243e7949b7f1bc924b15e 2024-11-21T21:25:52,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/77cf3db2b9c04cfb94538f379fa4b71c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/77cf3db2b9c04cfb94538f379fa4b71c 2024-11-21T21:25:52,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/866f8358cf1443339c3a56f4c7e22a08 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/866f8358cf1443339c3a56f4c7e22a08 2024-11-21T21:25:52,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/277d89bdcef243fe99857b5b2f9140a2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/277d89bdcef243fe99857b5b2f9140a2 2024-11-21T21:25:52,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/18d6af7d5ae5471582bd4cc159d0c3ae to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/18d6af7d5ae5471582bd4cc159d0c3ae 2024-11-21T21:25:52,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/dab4762355214ab7ad505d1c2bb308a9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/dab4762355214ab7ad505d1c2bb308a9 2024-11-21T21:25:52,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ee8ad6944b6e48a68a6af892ee73ebf0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ee8ad6944b6e48a68a6af892ee73ebf0 2024-11-21T21:25:52,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/3323368efd8b475f8085edeccd1765bc to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/3323368efd8b475f8085edeccd1765bc 2024-11-21T21:25:52,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/d03986302a2e4b41ba22ed329232b5ad to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/d03986302a2e4b41ba22ed329232b5ad 2024-11-21T21:25:52,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/2e776e3cb2114547b7bff924402d98b4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/2e776e3cb2114547b7bff924402d98b4 2024-11-21T21:25:52,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/0fe431059e7145dea4997abca1425f3b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/0fe431059e7145dea4997abca1425f3b 2024-11-21T21:25:52,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/b6937195b6bd44d58f3796facb16e9db to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/b6937195b6bd44d58f3796facb16e9db 2024-11-21T21:25:52,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f7100fd22fb943adb150a33deb05f089 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f7100fd22fb943adb150a33deb05f089 2024-11-21T21:25:52,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ccbc75b3ad694d81b389644999c040cd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/ccbc75b3ad694d81b389644999c040cd 2024-11-21T21:25:52,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e48d330bfb76482a95c3ef3c0bd760cb to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e48d330bfb76482a95c3ef3c0bd760cb 2024-11-21T21:25:52,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/7665f9b9d5734c4d8180fd8f438143a0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/7665f9b9d5734c4d8180fd8f438143a0 2024-11-21T21:25:52,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44e0a61fed2545c0b80651d588336a5e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/44e0a61fed2545c0b80651d588336a5e 2024-11-21T21:25:52,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e9c0bd89d6e449afadaa2d59d985d88b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e9c0bd89d6e449afadaa2d59d985d88b 2024-11-21T21:25:52,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e94e5ad96ff8492fb32a8ce4c7e78196 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/e94e5ad96ff8492fb32a8ce4c7e78196 2024-11-21T21:25:52,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/ae61227181024b96b11ebc044632da19, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a8db69c31b7a4965bb02182a71bd890d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3590f22abe444b8b9ef4777fc12d0128, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/dd42350630384033a2f6131fd2acf777, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/20a9f01a942548c7bda34f7d0550ac3d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3b4fbba1a9f4aff8bb277c51e266260, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/837d7eff133644a28989fd5f0b693404, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/5cf3b20b276a41c29dcd11a3efb4ccea, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d2eaa87024274a0687cab28bb07c4597, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e509032b7dc14a0395550928776ad2a2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/84bb5c8d5ac1465f880c7cb8bcf7bc27, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/55281abf3f4f4e2e956e308aa22755c7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3f93f033ac2c4625b44ccf1fcfbb90b3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/6115e850f548411b9d7e822fac7a46f6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/7d5ebd7c532d4ad4a731070578883b5c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d576a0b12909498388d391a1fb108f30, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e349e83ed7574e4c830b8e4ae73bff14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/efa3bfaaa2824799a30c5bdfaf43d57c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/4eabf5887aab43e49eaa3001cf7fd0b7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/98a174057e704ad88d9200bab7cb6a47] to archive 2024-11-21T21:25:52,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:25:52,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/ae61227181024b96b11ebc044632da19 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/ae61227181024b96b11ebc044632da19 2024-11-21T21:25:52,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a8db69c31b7a4965bb02182a71bd890d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a8db69c31b7a4965bb02182a71bd890d 2024-11-21T21:25:52,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3590f22abe444b8b9ef4777fc12d0128 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3590f22abe444b8b9ef4777fc12d0128 2024-11-21T21:25:52,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/dd42350630384033a2f6131fd2acf777 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/dd42350630384033a2f6131fd2acf777 2024-11-21T21:25:52,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/20a9f01a942548c7bda34f7d0550ac3d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/20a9f01a942548c7bda34f7d0550ac3d 2024-11-21T21:25:52,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3b4fbba1a9f4aff8bb277c51e266260 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3b4fbba1a9f4aff8bb277c51e266260 2024-11-21T21:25:52,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/837d7eff133644a28989fd5f0b693404 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/837d7eff133644a28989fd5f0b693404 2024-11-21T21:25:52,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/5cf3b20b276a41c29dcd11a3efb4ccea to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/5cf3b20b276a41c29dcd11a3efb4ccea 2024-11-21T21:25:52,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d2eaa87024274a0687cab28bb07c4597 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d2eaa87024274a0687cab28bb07c4597 2024-11-21T21:25:52,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e509032b7dc14a0395550928776ad2a2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e509032b7dc14a0395550928776ad2a2 2024-11-21T21:25:52,560 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/84bb5c8d5ac1465f880c7cb8bcf7bc27 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/84bb5c8d5ac1465f880c7cb8bcf7bc27 2024-11-21T21:25:52,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/55281abf3f4f4e2e956e308aa22755c7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/55281abf3f4f4e2e956e308aa22755c7 2024-11-21T21:25:52,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3f93f033ac2c4625b44ccf1fcfbb90b3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/3f93f033ac2c4625b44ccf1fcfbb90b3 2024-11-21T21:25:52,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/6115e850f548411b9d7e822fac7a46f6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/6115e850f548411b9d7e822fac7a46f6 2024-11-21T21:25:52,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/7d5ebd7c532d4ad4a731070578883b5c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/7d5ebd7c532d4ad4a731070578883b5c 2024-11-21T21:25:52,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d576a0b12909498388d391a1fb108f30 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/d576a0b12909498388d391a1fb108f30 2024-11-21T21:25:52,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/f3c0f0b75ab24b0fb26ea6e462ff3ac9 2024-11-21T21:25:52,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e349e83ed7574e4c830b8e4ae73bff14 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/e349e83ed7574e4c830b8e4ae73bff14 2024-11-21T21:25:52,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/efa3bfaaa2824799a30c5bdfaf43d57c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/efa3bfaaa2824799a30c5bdfaf43d57c 2024-11-21T21:25:52,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/4eabf5887aab43e49eaa3001cf7fd0b7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/4eabf5887aab43e49eaa3001cf7fd0b7 2024-11-21T21:25:52,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/98a174057e704ad88d9200bab7cb6a47 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/98a174057e704ad88d9200bab7cb6a47 2024-11-21T21:25:52,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f5c3819ac6f54f589fe9f1be38988a9e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/b27a050854af46eaa860e93a15ed1f68, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9171d837cba6493bbe3f283b6c1496e7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/0e3c57d90d574a399e93776b72abced8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f3136832134641519ba25ba2bc8585b1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/501c7da17e6b443b983b3913467a1f01, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/8acedc4ec00d44a190815d6bb03951a8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c79042f30c6b4362ad4342d9d3282f58, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f40ab871923b4e8e89fc3a5d2993a17f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/2095c84040d744c399f892ef9ea63f53, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/463c8da8e0fb4243853329ff6a42b97d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/cd1cccf4f8c3458c9b37d97b43146f25, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/ba929c69259341e3839018159ddd7b89, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/d84128a3ea2247deb98828f5fadde1da, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/eed49dc187754621aea9e916a21ddedd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e9b9f12b6def4cff80a7dcfe8ddcfe05, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e5d56fc506f84cd1b181bfd157dcfef1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/5983f318387e411fb73006fbafbc2923, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/486b380136dc47d5961d5291c9eab69f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/28878e1de9344456bb5772a019894423, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c6663e59f2124131949594ade4f18d9c] to archive 2024-11-21T21:25:52,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:25:52,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f5c3819ac6f54f589fe9f1be38988a9e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f5c3819ac6f54f589fe9f1be38988a9e 2024-11-21T21:25:52,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/b27a050854af46eaa860e93a15ed1f68 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/b27a050854af46eaa860e93a15ed1f68 2024-11-21T21:25:52,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9171d837cba6493bbe3f283b6c1496e7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9171d837cba6493bbe3f283b6c1496e7 2024-11-21T21:25:52,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/0e3c57d90d574a399e93776b72abced8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/0e3c57d90d574a399e93776b72abced8 2024-11-21T21:25:52,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f3136832134641519ba25ba2bc8585b1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f3136832134641519ba25ba2bc8585b1 2024-11-21T21:25:52,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/501c7da17e6b443b983b3913467a1f01 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/501c7da17e6b443b983b3913467a1f01 2024-11-21T21:25:52,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/8acedc4ec00d44a190815d6bb03951a8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/8acedc4ec00d44a190815d6bb03951a8 2024-11-21T21:25:52,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c79042f30c6b4362ad4342d9d3282f58 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c79042f30c6b4362ad4342d9d3282f58 2024-11-21T21:25:52,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f40ab871923b4e8e89fc3a5d2993a17f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/f40ab871923b4e8e89fc3a5d2993a17f 2024-11-21T21:25:52,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/2095c84040d744c399f892ef9ea63f53 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/2095c84040d744c399f892ef9ea63f53 2024-11-21T21:25:52,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/463c8da8e0fb4243853329ff6a42b97d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/463c8da8e0fb4243853329ff6a42b97d 2024-11-21T21:25:52,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/cd1cccf4f8c3458c9b37d97b43146f25 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/cd1cccf4f8c3458c9b37d97b43146f25 2024-11-21T21:25:52,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/ba929c69259341e3839018159ddd7b89 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/ba929c69259341e3839018159ddd7b89 2024-11-21T21:25:52,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/d84128a3ea2247deb98828f5fadde1da to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/d84128a3ea2247deb98828f5fadde1da 2024-11-21T21:25:52,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/eed49dc187754621aea9e916a21ddedd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/eed49dc187754621aea9e916a21ddedd 2024-11-21T21:25:52,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e9b9f12b6def4cff80a7dcfe8ddcfe05 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e9b9f12b6def4cff80a7dcfe8ddcfe05 2024-11-21T21:25:52,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e5d56fc506f84cd1b181bfd157dcfef1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/e5d56fc506f84cd1b181bfd157dcfef1 2024-11-21T21:25:52,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/5983f318387e411fb73006fbafbc2923 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/5983f318387e411fb73006fbafbc2923 2024-11-21T21:25:52,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/486b380136dc47d5961d5291c9eab69f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/486b380136dc47d5961d5291c9eab69f 2024-11-21T21:25:52,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/28878e1de9344456bb5772a019894423 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/28878e1de9344456bb5772a019894423 2024-11-21T21:25:52,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c6663e59f2124131949594ade4f18d9c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/c6663e59f2124131949594ade4f18d9c 2024-11-21T21:25:52,810 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/9df90ab8e4204973a41d753f8684dbb7 2024-11-21T21:25:52,820 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/a1e004975003405689c6cf81fdf9ad6f is 50, key is test_row_0/B:col10/1732224349370/Put/seqid=0 2024-11-21T21:25:52,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742285_1461 (size=12301) 2024-11-21T21:25:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:25:53,227 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/a1e004975003405689c6cf81fdf9ad6f 2024-11-21T21:25:53,237 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/09f6c1ab7205421b90809b67201bbf07 is 50, key is test_row_0/C:col10/1732224349370/Put/seqid=0 2024-11-21T21:25:53,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742286_1462 (size=12301) 2024-11-21T21:25:53,642 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/09f6c1ab7205421b90809b67201bbf07 2024-11-21T21:25:53,652 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/A/9df90ab8e4204973a41d753f8684dbb7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/9df90ab8e4204973a41d753f8684dbb7 2024-11-21T21:25:53,659 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/9df90ab8e4204973a41d753f8684dbb7, entries=150, sequenceid=353, filesize=30.5 K 2024-11-21T21:25:53,660 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/B/a1e004975003405689c6cf81fdf9ad6f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a1e004975003405689c6cf81fdf9ad6f 2024-11-21T21:25:53,664 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a1e004975003405689c6cf81fdf9ad6f, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T21:25:53,665 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/.tmp/C/09f6c1ab7205421b90809b67201bbf07 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/09f6c1ab7205421b90809b67201bbf07 2024-11-21T21:25:53,668 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/09f6c1ab7205421b90809b67201bbf07, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T21:25:53,669 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 58ad0d609f8c768bc6016e57e5d831ac in 1680ms, sequenceid=353, compaction requested=true 2024-11-21T21:25:53,674 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/recovered.edits/356.seqid, newMaxSeqId=356, maxSeqId=4 2024-11-21T21:25:53,675 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. 2024-11-21T21:25:53,675 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 58ad0d609f8c768bc6016e57e5d831ac: Waiting for close lock at 1732224351989Running coprocessor pre-close hooks at 1732224351989Disabling compacts and flushes for region at 1732224351989Disabling writes for close at 1732224351989Obtaining lock to block concurrent updates at 1732224351990 (+1 ms)Preparing flush snapshotting stores in 58ad0d609f8c768bc6016e57e5d831ac at 1732224351990Finished memstore snapshotting TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732224351990Flushing stores of TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac. at 1732224351991 (+1 ms)Flushing 58ad0d609f8c768bc6016e57e5d831ac/A: creating writer at 1732224351991Flushing mob file 58ad0d609f8c768bc6016e57e5d831ac/A: appending metadata at 1732224352000 (+9 ms)Flushing mob file 58ad0d609f8c768bc6016e57e5d831ac/A: closing flushed file at 1732224352000Flushing 58ad0d609f8c768bc6016e57e5d831ac/A: appending metadata at 1732224352407 (+407 ms)Flushing 58ad0d609f8c768bc6016e57e5d831ac/A: closing flushed file at 1732224352407Flushing 58ad0d609f8c768bc6016e57e5d831ac/B: creating writer at 1732224352816 (+409 ms)Flushing 58ad0d609f8c768bc6016e57e5d831ac/B: appending metadata at 1732224352820 (+4 ms)Flushing 58ad0d609f8c768bc6016e57e5d831ac/B: closing flushed file at 1732224352820Flushing 58ad0d609f8c768bc6016e57e5d831ac/C: creating writer at 1732224353233 (+413 ms)Flushing 58ad0d609f8c768bc6016e57e5d831ac/C: appending metadata at 1732224353237 (+4 ms)Flushing 58ad0d609f8c768bc6016e57e5d831ac/C: closing flushed file at 1732224353237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ffd978b: reopening flushed file at 1732224353651 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195c904f: reopening flushed file at 1732224353659 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41de3270: reopening flushed file at 1732224353664 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 58ad0d609f8c768bc6016e57e5d831ac in 1680ms, sequenceid=353, compaction requested=true at 1732224353669 (+5 ms)Writing region close event to WAL at 1732224353671 (+2 ms)Running coprocessor post-close hooks at 1732224353674 (+3 ms)Closed at 1732224353675 (+1 ms) 2024-11-21T21:25:53,677 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:53,677 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=58ad0d609f8c768bc6016e57e5d831ac, regionState=CLOSED 2024-11-21T21:25:53,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:25:53,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T21:25:53,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 58ad0d609f8c768bc6016e57e5d831ac, server=849dee24a242,34123,1732224229084 in 1.8450 sec 2024-11-21T21:25:53,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-21T21:25:53,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58ad0d609f8c768bc6016e57e5d831ac, UNASSIGN in 1.8510 sec 2024-11-21T21:25:53,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T21:25:53,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8560 sec 2024-11-21T21:25:53,690 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224353690"}]},"ts":"1732224353690"} 2024-11-21T21:25:53,691 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:25:53,691 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:25:53,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8700 sec 2024-11-21T21:25:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:25:53,971 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:53,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:25:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:53,978 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T21:25:53,980 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:53,982 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:53,984 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/recovered.edits] 2024-11-21T21:25:53,986 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/9df90ab8e4204973a41d753f8684dbb7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/9df90ab8e4204973a41d753f8684dbb7 2024-11-21T21:25:53,988 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/c7d6ea0ccb444a9799230d976a05ab1d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/c7d6ea0ccb444a9799230d976a05ab1d 2024-11-21T21:25:53,989 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f4f2639dfbff4ff1a2fb00f5ad98c776 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/A/f4f2639dfbff4ff1a2fb00f5ad98c776 2024-11-21T21:25:53,991 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/62a82309622b42488b605d8ae6afeaa4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/62a82309622b42488b605d8ae6afeaa4 2024-11-21T21:25:53,992 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/88b51d4959b8412989e7df189c1176cf to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/88b51d4959b8412989e7df189c1176cf 2024-11-21T21:25:53,993 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a1e004975003405689c6cf81fdf9ad6f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/B/a1e004975003405689c6cf81fdf9ad6f 2024-11-21T21:25:53,995 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/09f6c1ab7205421b90809b67201bbf07 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/09f6c1ab7205421b90809b67201bbf07 2024-11-21T21:25:53,997 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/41b10208889e4ea18fa7b13b92fd0863 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/41b10208889e4ea18fa7b13b92fd0863 2024-11-21T21:25:53,998 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9414db46a8094d2e8e9d3f38b8cb9c7a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/C/9414db46a8094d2e8e9d3f38b8cb9c7a 2024-11-21T21:25:54,001 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/recovered.edits/356.seqid to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac/recovered.edits/356.seqid 2024-11-21T21:25:54,002 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,002 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:25:54,002 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:25:54,003 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T21:25:54,006 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121295756079dc74f28aa688c6871f6a350_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121295756079dc74f28aa688c6871f6a350_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,008 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215c5d70cfce5742a08b283937795ac1bc_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215c5d70cfce5742a08b283937795ac1bc_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,009 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121634ece2ef08a44c4abc433f36e32ec8a_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121634ece2ef08a44c4abc433f36e32ec8a_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,011 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121687c944fc50f47eda2f27427ee91cf38_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121687c944fc50f47eda2f27427ee91cf38_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217872de3ad45741ac85d9a6167cbeea64_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217872de3ad45741ac85d9a6167cbeea64_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112178b0aac648514e5ab33a8cc9352d95cd_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112178b0aac648514e5ab33a8cc9352d95cd_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218919caceefba41e0b4f3d05855a614a1_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218919caceefba41e0b4f3d05855a614a1_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,018 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112193eda7937596498da2bf6b3cd3b2ac7f_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112193eda7937596498da2bf6b3cd3b2ac7f_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b65f43935f6c48dfb93147e8b9b71bb0_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b65f43935f6c48dfb93147e8b9b71bb0_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,022 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b810027bfd6e429db6b00575020fe216_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b810027bfd6e429db6b00575020fe216_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,024 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bbafa11903244861a54960e0f2287e09_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bbafa11903244861a54960e0f2287e09_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,026 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4cbea75e7f3456db0db9a02c12024d0_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4cbea75e7f3456db0db9a02c12024d0_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c59447db78274d0fab6dc74a0e953b40_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c59447db78274d0fab6dc74a0e953b40_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c5a513ef8a594c6f9e9dd4cc63a10af4_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c5a513ef8a594c6f9e9dd4cc63a10af4_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,032 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c661ff3561b44ae19804f510b49a1f90_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c661ff3561b44ae19804f510b49a1f90_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ce2c5ca94af240edaffb495e37a880df_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ce2c5ca94af240edaffb495e37a880df_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d514e075761d4a87bec88571d97ee203_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d514e075761d4a87bec88571d97ee203_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6dcbc7297c54b9d921b29ce11d8077f_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6dcbc7297c54b9d921b29ce11d8077f_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,040 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f4867da266e9462eaa9e68c6cfe32a23_58ad0d609f8c768bc6016e57e5d831ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f4867da266e9462eaa9e68c6cfe32a23_58ad0d609f8c768bc6016e57e5d831ac 2024-11-21T21:25:54,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:25:54,045 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:54,048 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:25:54,051 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:25:54,052 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:54,052 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:25:54,052 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732224354052"}]},"ts":"9223372036854775807"} 2024-11-21T21:25:54,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:25:54,055 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 58ad0d609f8c768bc6016e57e5d831ac, NAME => 'TestAcidGuarantees,,1732224326444.58ad0d609f8c768bc6016e57e5d831ac.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:25:54,055 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:25:54,055 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732224354055"}]},"ts":"9223372036854775807"} 2024-11-21T21:25:54,057 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:25:54,058 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:54,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-21T21:25:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T21:25:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:25:54,090 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:54,103 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1390536951_22 at /127.0.0.1:39352 [Waiting for operation #1311] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-185090283_22 at /127.0.0.1:33686 [Waiting for operation #1260] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1390536951_22 at /127.0.0.1:59892 [Waiting for operation #891] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-185090283_22 at /127.0.0.1:59870 [Waiting for operation #980] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=489 (was 448) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11621 (was 12581) 2024-11-21T21:25:54,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=11621 2024-11-21T21:25:54,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:25:54,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:25:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:25:54,115 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:25:54,116 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:54,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-21T21:25:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T21:25:54,117 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:25:54,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742287_1463 (size=959) 2024-11-21T21:25:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T21:25:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T21:25:54,525 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:25:54,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742288_1464 (size=53) 2024-11-21T21:25:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T21:25:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:25:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c74aaba9f7218f5ab8983cd7c853d241, disabling compactions & flushes 2024-11-21T21:25:54,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. after waiting 0 ms 2024-11-21T21:25:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:54,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c74aaba9f7218f5ab8983cd7c853d241: Waiting for close lock at 1732224354934Disabling compacts and flushes for region at 1732224354934Disabling writes for close at 1732224354934Writing region close event to WAL at 1732224354934Closed at 1732224354934 2024-11-21T21:25:54,935 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:25:54,935 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732224354935"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732224354935"}]},"ts":"1732224354935"} 2024-11-21T21:25:54,936 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:25:54,937 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:25:54,937 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224354937"}]},"ts":"1732224354937"} 2024-11-21T21:25:54,938 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:25:54,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, ASSIGN}] 2024-11-21T21:25:54,939 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, ASSIGN 2024-11-21T21:25:54,940 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, ASSIGN; state=OFFLINE, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=false 2024-11-21T21:25:55,091 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=c74aaba9f7218f5ab8983cd7c853d241, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:55,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, ASSIGN because future has completed 2024-11-21T21:25:55,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:25:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T21:25:55,296 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:55,296 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:25:55,298 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,298 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:25:55,298 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,298 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,300 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,301 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c74aaba9f7218f5ab8983cd7c853d241 columnFamilyName A 2024-11-21T21:25:55,301 DEBUG [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:55,302 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] regionserver.HStore(327): Store=c74aaba9f7218f5ab8983cd7c853d241/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:55,302 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,303 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c74aaba9f7218f5ab8983cd7c853d241 columnFamilyName B 2024-11-21T21:25:55,303 DEBUG [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:55,304 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] regionserver.HStore(327): Store=c74aaba9f7218f5ab8983cd7c853d241/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:55,304 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,305 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c74aaba9f7218f5ab8983cd7c853d241 columnFamilyName C 2024-11-21T21:25:55,305 DEBUG [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:25:55,305 INFO [StoreOpener-c74aaba9f7218f5ab8983cd7c853d241-1 {}] regionserver.HStore(327): Store=c74aaba9f7218f5ab8983cd7c853d241/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:25:55,305 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,306 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,306 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,308 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,308 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,308 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:25:55,309 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,311 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:25:55,312 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened c74aaba9f7218f5ab8983cd7c853d241; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60429761, jitterRate=-0.09952639043331146}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:25:55,312 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:55,313 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for c74aaba9f7218f5ab8983cd7c853d241: Running coprocessor pre-open hook at 1732224355298Writing region info on filesystem at 1732224355299 (+1 ms)Initializing all the Stores at 1732224355299Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224355299Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224355300 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224355300Cleaning up temporary data from old regions at 1732224355308 (+8 ms)Running coprocessor post-open hooks at 1732224355312 (+4 ms)Region opened successfully at 1732224355313 (+1 ms) 2024-11-21T21:25:55,314 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., pid=122, masterSystemTime=1732224355289 2024-11-21T21:25:55,316 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:55,316 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:55,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=c74aaba9f7218f5ab8983cd7c853d241, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:25:55,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:25:55,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-21T21:25:55,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 in 224 msec 2024-11-21T21:25:55,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-21T21:25:55,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, ASSIGN in 384 msec 2024-11-21T21:25:55,326 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:25:55,326 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224355326"}]},"ts":"1732224355326"} 2024-11-21T21:25:55,328 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:25:55,329 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:25:55,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2160 sec 2024-11-21T21:25:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T21:25:56,261 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:25:56,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,265 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520bec18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,269 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a37fbb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,270 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,273 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:25:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,35047,1732224227985 2024-11-21T21:25:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b81669e 2024-11-21T21:25:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:25:56,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:25:56,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,277 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d92657, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,278 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c382cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3864ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,280 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670811e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,282 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427a330c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,283 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76639cc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,284 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f346270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,286 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a2eba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,288 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec169f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,289 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ceff97f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,291 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0068bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d6b5f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2033adc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f76f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,298 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45120b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b45e14a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ee366d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c0eac5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:25:56,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:25:56,303 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:25:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:25:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:25:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c277f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:25:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:25:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:25:56,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:25:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1ecc19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:25:56,305 DEBUG [Thread-2089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,305 DEBUG [Thread-2078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,305 DEBUG [Thread-2082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,305 DEBUG [Thread-2093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,306 DEBUG [Thread-2091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,306 DEBUG [Thread-2074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,306 DEBUG [Thread-2076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,308 DEBUG [Thread-2080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,308 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,308 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:25:56,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,309 DEBUG [Thread-2080 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,309 DEBUG [Thread-2080 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,309 DEBUG [Thread-2087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:25:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-21T21:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:25:56,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T21:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:25:56,312 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:25:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,313 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:25:56,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2] 2024-11-21T21:25:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:56,319 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:56,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:25:56,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/262a383815d940a2bc35581d4c84c2d5 is 50, key is test_row_0/A:col10/1732224356318/Put/seqid=0 2024-11-21T21:25:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742289_1465 (size=12001) 2024-11-21T21:25:56,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/262a383815d940a2bc35581d4c84c2d5 2024-11-21T21:25:56,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224416361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224416367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224416373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224416375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224416373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/4ce1d3328f8b42e781d88dbe6cdb7c5a is 50, key is test_row_0/B:col10/1732224356318/Put/seqid=0 2024-11-21T21:25:56,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742290_1466 (size=12001) 2024-11-21T21:25:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T21:25:56,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T21:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224416480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224416480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224416489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224416499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224416499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T21:25:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:25:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T21:25:56,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224416689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224416689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224416699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224416710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224416710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:56,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T21:25:56,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:25:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/4ce1d3328f8b42e781d88dbe6cdb7c5a 2024-11-21T21:25:56,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/401f381aae03429289d5ddcdc29624a4 is 50, key is test_row_0/C:col10/1732224356318/Put/seqid=0 2024-11-21T21:25:56,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742291_1467 (size=12001) 2024-11-21T21:25:56,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T21:25:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:25:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:56,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T21:25:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224417000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224417000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224417009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224417019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224417019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T21:25:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:25:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:57,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:57,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/401f381aae03429289d5ddcdc29624a4 2024-11-21T21:25:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/262a383815d940a2bc35581d4c84c2d5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/262a383815d940a2bc35581d4c84c2d5 2024-11-21T21:25:57,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T21:25:57,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:57,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:25:57,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:57,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:57,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:25:57,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/262a383815d940a2bc35581d4c84c2d5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:25:57,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/4ce1d3328f8b42e781d88dbe6cdb7c5a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ce1d3328f8b42e781d88dbe6cdb7c5a 2024-11-21T21:25:57,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ce1d3328f8b42e781d88dbe6cdb7c5a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:25:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/401f381aae03429289d5ddcdc29624a4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/401f381aae03429289d5ddcdc29624a4 2024-11-21T21:25:57,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/401f381aae03429289d5ddcdc29624a4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:25:57,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c74aaba9f7218f5ab8983cd7c853d241 in 921ms, sequenceid=14, compaction requested=false 2024-11-21T21:25:57,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:25:57,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T21:25:57,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:57,384 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:25:57,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/2bd516886ad54bd1ad7aa863162e3bbf is 50, key is test_row_0/A:col10/1732224356370/Put/seqid=0 2024-11-21T21:25:57,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742292_1468 (size=12001) 2024-11-21T21:25:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T21:25:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:25:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:57,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224417528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224417529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224417529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224417533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224417534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224417650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224417650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224417650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,792 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/2bd516886ad54bd1ad7aa863162e3bbf 2024-11-21T21:25:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/b90cbdff6dc544bf8ba5b4ae8ce691ac is 50, key is test_row_0/B:col10/1732224356370/Put/seqid=0 2024-11-21T21:25:57,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742293_1469 (size=12001) 2024-11-21T21:25:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224417860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224417860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224417860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224418169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224418169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224418169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,201 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/b90cbdff6dc544bf8ba5b4ae8ce691ac 2024-11-21T21:25:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/be64eda2de384a95a8c45d47b7f14cfa is 50, key is test_row_0/C:col10/1732224356370/Put/seqid=0 2024-11-21T21:25:58,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742294_1470 (size=12001) 2024-11-21T21:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T21:25:58,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224418549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224418549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,554 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,554 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,610 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/be64eda2de384a95a8c45d47b7f14cfa 2024-11-21T21:25:58,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/2bd516886ad54bd1ad7aa863162e3bbf as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/2bd516886ad54bd1ad7aa863162e3bbf 2024-11-21T21:25:58,616 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/2bd516886ad54bd1ad7aa863162e3bbf, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T21:25:58,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/b90cbdff6dc544bf8ba5b4ae8ce691ac as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b90cbdff6dc544bf8ba5b4ae8ce691ac 2024-11-21T21:25:58,620 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b90cbdff6dc544bf8ba5b4ae8ce691ac, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T21:25:58,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/be64eda2de384a95a8c45d47b7f14cfa as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/be64eda2de384a95a8c45d47b7f14cfa 2024-11-21T21:25:58,623 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/be64eda2de384a95a8c45d47b7f14cfa, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T21:25:58,624 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c74aaba9f7218f5ab8983cd7c853d241 in 1239ms, sequenceid=37, compaction requested=false 2024-11-21T21:25:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:25:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-21T21:25:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-21T21:25:58,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T21:25:58,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3120 sec 2024-11-21T21:25:58,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.3190 sec 2024-11-21T21:25:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:58,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T21:25:58,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/0d021bbf24294ca398f123910b40a533 is 50, key is test_row_0/A:col10/1732224357522/Put/seqid=0 2024-11-21T21:25:58,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742295_1471 (size=14341) 2024-11-21T21:25:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224418731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224418732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224418733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224418849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224418850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:58,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224418850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:58,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:58,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224419059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224419059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224419060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/0d021bbf24294ca398f123910b40a533 2024-11-21T21:25:59,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2826b9c9574e4cc68b41a42102f0ebba is 50, key is test_row_0/B:col10/1732224357522/Put/seqid=0 2024-11-21T21:25:59,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742296_1472 (size=12001) 2024-11-21T21:25:59,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2826b9c9574e4cc68b41a42102f0ebba 2024-11-21T21:25:59,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c614e602a04d41a4b501e5b0482c9a71 is 50, key is test_row_0/C:col10/1732224357522/Put/seqid=0 2024-11-21T21:25:59,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742297_1473 (size=12001) 2024-11-21T21:25:59,201 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:25:59,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224419369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224419369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224419369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c614e602a04d41a4b501e5b0482c9a71 2024-11-21T21:25:59,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/0d021bbf24294ca398f123910b40a533 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0d021bbf24294ca398f123910b40a533 2024-11-21T21:25:59,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0d021bbf24294ca398f123910b40a533, entries=200, sequenceid=51, filesize=14.0 K 2024-11-21T21:25:59,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2826b9c9574e4cc68b41a42102f0ebba as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2826b9c9574e4cc68b41a42102f0ebba 2024-11-21T21:25:59,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2826b9c9574e4cc68b41a42102f0ebba, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T21:25:59,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c614e602a04d41a4b501e5b0482c9a71 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c614e602a04d41a4b501e5b0482c9a71 2024-11-21T21:25:59,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c614e602a04d41a4b501e5b0482c9a71, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T21:25:59,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c74aaba9f7218f5ab8983cd7c853d241 in 855ms, sequenceid=51, compaction requested=true 2024-11-21T21:25:59,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:25:59,540 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:59,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:25:59,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:59,540 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:59,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:25:59,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:59,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:25:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:59,541 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:59,541 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:59,541 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:25:59,541 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:25:59,541 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:59,541 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:59,541 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/262a383815d940a2bc35581d4c84c2d5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/2bd516886ad54bd1ad7aa863162e3bbf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0d021bbf24294ca398f123910b40a533] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=37.4 K 2024-11-21T21:25:59,541 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ce1d3328f8b42e781d88dbe6cdb7c5a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b90cbdff6dc544bf8ba5b4ae8ce691ac, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2826b9c9574e4cc68b41a42102f0ebba] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=35.2 K 2024-11-21T21:25:59,541 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce1d3328f8b42e781d88dbe6cdb7c5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732224356317 2024-11-21T21:25:59,541 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262a383815d940a2bc35581d4c84c2d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732224356317 2024-11-21T21:25:59,542 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b90cbdff6dc544bf8ba5b4ae8ce691ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732224356360 2024-11-21T21:25:59,542 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd516886ad54bd1ad7aa863162e3bbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732224356360 2024-11-21T21:25:59,542 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2826b9c9574e4cc68b41a42102f0ebba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224357522 2024-11-21T21:25:59,542 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d021bbf24294ca398f123910b40a533, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224357522 2024-11-21T21:25:59,557 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:59,557 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:59,557 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/59f10e1a8f3b445ab4bdb2146e6c1f97 is 50, key is test_row_0/B:col10/1732224357522/Put/seqid=0 2024-11-21T21:25:59,557 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/99748379a007480ca664056756be79c1 is 50, key is test_row_0/A:col10/1732224357522/Put/seqid=0 2024-11-21T21:25:59,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742299_1475 (size=12104) 2024-11-21T21:25:59,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742298_1474 (size=12104) 2024-11-21T21:25:59,578 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/59f10e1a8f3b445ab4bdb2146e6c1f97 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/59f10e1a8f3b445ab4bdb2146e6c1f97 2024-11-21T21:25:59,583 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 59f10e1a8f3b445ab4bdb2146e6c1f97(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:59,583 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:25:59,583 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224359540; duration=0sec 2024-11-21T21:25:59,583 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:25:59,583 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:25:59,583 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:25:59,584 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:25:59,584 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:25:59,584 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:25:59,584 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/401f381aae03429289d5ddcdc29624a4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/be64eda2de384a95a8c45d47b7f14cfa, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c614e602a04d41a4b501e5b0482c9a71] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=35.2 K 2024-11-21T21:25:59,585 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 401f381aae03429289d5ddcdc29624a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732224356317 2024-11-21T21:25:59,585 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting be64eda2de384a95a8c45d47b7f14cfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732224356360 2024-11-21T21:25:59,585 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c614e602a04d41a4b501e5b0482c9a71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224357522 2024-11-21T21:25:59,593 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:25:59,594 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/26a44ba830dd4f57b788dee96b9728c8 is 50, key is test_row_0/C:col10/1732224357522/Put/seqid=0 2024-11-21T21:25:59,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742300_1476 (size=12104) 2024-11-21T21:25:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:25:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:25:59,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:25:59,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c3fab95373da41ff9cee8109964df3ee is 50, key is test_row_0/A:col10/1732224359892/Put/seqid=0 2024-11-21T21:25:59,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742301_1477 (size=14341) 2024-11-21T21:25:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224419910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224419912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:25:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224419918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:25:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:25:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:25:59,970 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/99748379a007480ca664056756be79c1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/99748379a007480ca664056756be79c1 2024-11-21T21:25:59,974 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 99748379a007480ca664056756be79c1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:25:59,974 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:25:59,974 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224359539; duration=0sec 2024-11-21T21:25:59,974 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:25:59,974 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:00,004 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/26a44ba830dd4f57b788dee96b9728c8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/26a44ba830dd4f57b788dee96b9728c8 2024-11-21T21:26:00,007 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into 26a44ba830dd4f57b788dee96b9728c8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:00,007 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:00,007 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224359540; duration=0sec 2024-11-21T21:26:00,007 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:00,007 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224420030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224420030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224420030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224420239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224420240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224420240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c3fab95373da41ff9cee8109964df3ee 2024-11-21T21:26:00,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/eaa2273b764c441f8448f3e26f7eba68 is 50, key is test_row_0/B:col10/1732224359892/Put/seqid=0 2024-11-21T21:26:00,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742302_1478 (size=12001) 2024-11-21T21:26:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T21:26:00,470 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:00,470 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T21:26:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:26:00,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:00,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:00,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224420559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224420559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224420559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,564 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224420559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224420570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:00,575 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:26:00,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:26:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:00,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:00,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/eaa2273b764c441f8448f3e26f7eba68 2024-11-21T21:26:00,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/64d27833434a40099a1c529dee6ced4c is 50, key is test_row_0/C:col10/1732224359892/Put/seqid=0 2024-11-21T21:26:00,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742303_1479 (size=12001) 2024-11-21T21:26:00,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/64d27833434a40099a1c529dee6ced4c 2024-11-21T21:26:00,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c3fab95373da41ff9cee8109964df3ee as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c3fab95373da41ff9cee8109964df3ee 2024-11-21T21:26:00,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c3fab95373da41ff9cee8109964df3ee, entries=200, sequenceid=75, filesize=14.0 K 2024-11-21T21:26:00,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/eaa2273b764c441f8448f3e26f7eba68 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/eaa2273b764c441f8448f3e26f7eba68 2024-11-21T21:26:00,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/eaa2273b764c441f8448f3e26f7eba68, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T21:26:00,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/64d27833434a40099a1c529dee6ced4c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/64d27833434a40099a1c529dee6ced4c 2024-11-21T21:26:00,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/64d27833434a40099a1c529dee6ced4c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T21:26:00,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c74aaba9f7218f5ab8983cd7c853d241 in 845ms, sequenceid=75, compaction requested=false 2024-11-21T21:26:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:00,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:26:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:00,777 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:26:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/fcc5f5b448374bb586b6a556c2f633eb is 50, key is test_row_0/A:col10/1732224359904/Put/seqid=0 2024-11-21T21:26:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:26:00,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742304_1480 (size=12001) 2024-11-21T21:26:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:01,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:26:01,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224421116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224421118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224421119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,188 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/fcc5f5b448374bb586b6a556c2f633eb 2024-11-21T21:26:01,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/868c147f4c404c438412084cffc1b3c2 is 50, key is test_row_0/B:col10/1732224359904/Put/seqid=0 2024-11-21T21:26:01,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742305_1481 (size=12001) 2024-11-21T21:26:01,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224421229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224421229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224421229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224421439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224421439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224421439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,600 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/868c147f4c404c438412084cffc1b3c2 2024-11-21T21:26:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:26:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/69dab6f58c9b491c8dccf5db73d58bb0 is 50, key is test_row_0/C:col10/1732224359904/Put/seqid=0 2024-11-21T21:26:01,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742306_1482 (size=12001) 2024-11-21T21:26:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224421749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224421749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224421750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,018 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/69dab6f58c9b491c8dccf5db73d58bb0 2024-11-21T21:26:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/fcc5f5b448374bb586b6a556c2f633eb as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/fcc5f5b448374bb586b6a556c2f633eb 2024-11-21T21:26:02,026 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/fcc5f5b448374bb586b6a556c2f633eb, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T21:26:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/868c147f4c404c438412084cffc1b3c2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/868c147f4c404c438412084cffc1b3c2 2024-11-21T21:26:02,030 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/868c147f4c404c438412084cffc1b3c2, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T21:26:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/69dab6f58c9b491c8dccf5db73d58bb0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/69dab6f58c9b491c8dccf5db73d58bb0 2024-11-21T21:26:02,037 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/69dab6f58c9b491c8dccf5db73d58bb0, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T21:26:02,038 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c74aaba9f7218f5ab8983cd7c853d241 in 1261ms, sequenceid=90, compaction requested=true 2024-11-21T21:26:02,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:02,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:02,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T21:26:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T21:26:02,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T21:26:02,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-21T21:26:02,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5710 sec 2024-11-21T21:26:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:02,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:26:02,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/b47015b761da42e886c60e6c1c88d973 is 50, key is test_row_0/A:col10/1732224361118/Put/seqid=0 2024-11-21T21:26:02,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742307_1483 (size=14341) 2024-11-21T21:26:02,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/b47015b761da42e886c60e6c1c88d973 2024-11-21T21:26:02,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/a2e44eb31e1746ffb3b5f57d92ce5632 is 50, key is test_row_0/B:col10/1732224361118/Put/seqid=0 2024-11-21T21:26:02,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224422302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224422304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224422315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742308_1484 (size=12001) 2024-11-21T21:26:02,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224422419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224422420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224422430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:26:02,620 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:02,620 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T21:26:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:26:02,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:02,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:02,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:02,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224422629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224422641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224422641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:02,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/a2e44eb31e1746ffb3b5f57d92ce5632 2024-11-21T21:26:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:26:02,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/1690a4d6ad22451aa6ea1d7bf4108514 is 50, key is test_row_0/C:col10/1732224361118/Put/seqid=0 2024-11-21T21:26:02,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742309_1485 (size=12001) 2024-11-21T21:26:02,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/1690a4d6ad22451aa6ea1d7bf4108514 2024-11-21T21:26:02,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/b47015b761da42e886c60e6c1c88d973 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b47015b761da42e886c60e6c1c88d973 2024-11-21T21:26:02,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b47015b761da42e886c60e6c1c88d973, entries=200, sequenceid=114, filesize=14.0 K 2024-11-21T21:26:02,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/a2e44eb31e1746ffb3b5f57d92ce5632 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a2e44eb31e1746ffb3b5f57d92ce5632 2024-11-21T21:26:02,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a2e44eb31e1746ffb3b5f57d92ce5632, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:26:02,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/1690a4d6ad22451aa6ea1d7bf4108514 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/1690a4d6ad22451aa6ea1d7bf4108514 2024-11-21T21:26:02,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/1690a4d6ad22451aa6ea1d7bf4108514, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:26:02,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c74aaba9f7218f5ab8983cd7c853d241 in 503ms, sequenceid=114, compaction requested=true 2024-11-21T21:26:02,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:02,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:02,771 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:26:02,771 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:02,771 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:02,771 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:02,772 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:02,772 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/99748379a007480ca664056756be79c1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c3fab95373da41ff9cee8109964df3ee, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/fcc5f5b448374bb586b6a556c2f633eb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b47015b761da42e886c60e6c1c88d973] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=51.5 K 2024-11-21T21:26:02,772 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99748379a007480ca664056756be79c1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224357522 2024-11-21T21:26:02,772 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3fab95373da41ff9cee8109964df3ee, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732224358718 2024-11-21T21:26:02,773 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:02,773 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc5f5b448374bb586b6a556c2f633eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732224359904 2024-11-21T21:26:02,773 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:02,773 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:02,773 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/59f10e1a8f3b445ab4bdb2146e6c1f97, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/eaa2273b764c441f8448f3e26f7eba68, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/868c147f4c404c438412084cffc1b3c2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a2e44eb31e1746ffb3b5f57d92ce5632] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=47.0 K 2024-11-21T21:26:02,773 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b47015b761da42e886c60e6c1c88d973, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224361104 2024-11-21T21:26:02,773 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f10e1a8f3b445ab4bdb2146e6c1f97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224357522 2024-11-21T21:26:02,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T21:26:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:02,774 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:26:02,774 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa2273b764c441f8448f3e26f7eba68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732224358718 2024-11-21T21:26:02,775 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 868c147f4c404c438412084cffc1b3c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732224359904 2024-11-21T21:26:02,775 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e44eb31e1746ffb3b5f57d92ce5632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224361104 2024-11-21T21:26:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/f74b475c27f041a387d220ed8b402674 is 50, key is test_row_0/A:col10/1732224362292/Put/seqid=0 2024-11-21T21:26:02,783 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:02,784 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/78add8f42ecf442ba77b2b9e94994ce9 is 50, key is test_row_0/A:col10/1732224361118/Put/seqid=0 2024-11-21T21:26:02,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742310_1486 (size=12001) 2024-11-21T21:26:02,785 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/f74b475c27f041a387d220ed8b402674 2024-11-21T21:26:02,790 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:02,790 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/5dfaf3e7713d4a68bd59ffc9988e1b53 is 50, key is test_row_0/B:col10/1732224361118/Put/seqid=0 2024-11-21T21:26:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/b44b0eedce6140059bb312ebf3b6e436 is 50, key is test_row_0/B:col10/1732224362292/Put/seqid=0 2024-11-21T21:26:02,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742311_1487 (size=12241) 2024-11-21T21:26:02,810 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/78add8f42ecf442ba77b2b9e94994ce9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/78add8f42ecf442ba77b2b9e94994ce9 2024-11-21T21:26:02,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742312_1488 (size=12241) 2024-11-21T21:26:02,817 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/5dfaf3e7713d4a68bd59ffc9988e1b53 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5dfaf3e7713d4a68bd59ffc9988e1b53 2024-11-21T21:26:02,817 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 78add8f42ecf442ba77b2b9e94994ce9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:02,817 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:02,817 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=12, startTime=1732224362770; duration=0sec 2024-11-21T21:26:02,817 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:02,817 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:02,817 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:02,819 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:02,819 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:02,819 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:02,819 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/26a44ba830dd4f57b788dee96b9728c8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/64d27833434a40099a1c529dee6ced4c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/69dab6f58c9b491c8dccf5db73d58bb0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/1690a4d6ad22451aa6ea1d7bf4108514] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=47.0 K 2024-11-21T21:26:02,819 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26a44ba830dd4f57b788dee96b9728c8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224357522 2024-11-21T21:26:02,820 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d27833434a40099a1c529dee6ced4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732224358718 2024-11-21T21:26:02,823 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 5dfaf3e7713d4a68bd59ffc9988e1b53(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:02,823 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:02,823 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=12, startTime=1732224362771; duration=0sec 2024-11-21T21:26:02,823 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:02,823 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:02,823 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69dab6f58c9b491c8dccf5db73d58bb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732224359904 2024-11-21T21:26:02,824 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1690a4d6ad22451aa6ea1d7bf4108514, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224361104 2024-11-21T21:26:02,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742313_1489 (size=12001) 2024-11-21T21:26:02,836 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:02,836 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c91e7a4f02a142c980b384fb62cb125f is 50, key is test_row_0/C:col10/1732224361118/Put/seqid=0 2024-11-21T21:26:02,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742314_1490 (size=12241) 2024-11-21T21:26:02,866 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c91e7a4f02a142c980b384fb62cb125f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c91e7a4f02a142c980b384fb62cb125f 2024-11-21T21:26:02,871 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into c91e7a4f02a142c980b384fb62cb125f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:02,871 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:02,871 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=12, startTime=1732224362771; duration=0sec 2024-11-21T21:26:02,871 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:02,871 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:26:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:02,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224422997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224423003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224423004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224423109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224423109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224423120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,226 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/b44b0eedce6140059bb312ebf3b6e436 2024-11-21T21:26:03,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/98bac4cb3e464dd0aa12ca5471b9330e is 50, key is test_row_0/C:col10/1732224362292/Put/seqid=0 2024-11-21T21:26:03,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742315_1491 (size=12001) 2024-11-21T21:26:03,247 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/98bac4cb3e464dd0aa12ca5471b9330e 2024-11-21T21:26:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:26:03,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/f74b475c27f041a387d220ed8b402674 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/f74b475c27f041a387d220ed8b402674 2024-11-21T21:26:03,254 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/f74b475c27f041a387d220ed8b402674, entries=150, sequenceid=126, filesize=11.7 K 2024-11-21T21:26:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/b44b0eedce6140059bb312ebf3b6e436 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b44b0eedce6140059bb312ebf3b6e436 2024-11-21T21:26:03,260 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b44b0eedce6140059bb312ebf3b6e436, entries=150, sequenceid=126, filesize=11.7 K 2024-11-21T21:26:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/98bac4cb3e464dd0aa12ca5471b9330e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/98bac4cb3e464dd0aa12ca5471b9330e 2024-11-21T21:26:03,264 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/98bac4cb3e464dd0aa12ca5471b9330e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-21T21:26:03,265 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c74aaba9f7218f5ab8983cd7c853d241 in 491ms, sequenceid=126, compaction requested=false 2024-11-21T21:26:03,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:03,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:03,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T21:26:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T21:26:03,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T21:26:03,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 645 msec 2024-11-21T21:26:03,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 648 msec 2024-11-21T21:26:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:03,323 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:03,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:26:03,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/d0c47964d306434e99d2df5e8e4a6e4f is 50, key is test_row_0/A:col10/1732224363322/Put/seqid=0 2024-11-21T21:26:03,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742316_1492 (size=14541) 2024-11-21T21:26:03,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/d0c47964d306434e99d2df5e8e4a6e4f 2024-11-21T21:26:03,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/8c219e19c59a476e85958fd8db8260c3 is 50, key is test_row_0/B:col10/1732224363322/Put/seqid=0 2024-11-21T21:26:03,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742317_1493 (size=12151) 2024-11-21T21:26:03,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224423341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224423345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224423348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224423460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224423460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224423460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224423670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224423670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224423679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/8c219e19c59a476e85958fd8db8260c3 2024-11-21T21:26:03,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/84d0e3f7727d4512b9f98548de45591a is 50, key is test_row_0/C:col10/1732224363322/Put/seqid=0 2024-11-21T21:26:03,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742318_1494 (size=12151) 2024-11-21T21:26:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:26:03,760 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:03,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T21:26:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:26:03,763 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:03,763 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:03,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:26:03,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T21:26:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:03,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:03,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:03,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224423990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224423990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224423990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T21:26:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:04,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:26:04,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/84d0e3f7727d4512b9f98548de45591a 2024-11-21T21:26:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/d0c47964d306434e99d2df5e8e4a6e4f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d0c47964d306434e99d2df5e8e4a6e4f 2024-11-21T21:26:04,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d0c47964d306434e99d2df5e8e4a6e4f, entries=200, sequenceid=154, filesize=14.2 K 2024-11-21T21:26:04,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/8c219e19c59a476e85958fd8db8260c3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8c219e19c59a476e85958fd8db8260c3 2024-11-21T21:26:04,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8c219e19c59a476e85958fd8db8260c3, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T21:26:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/84d0e3f7727d4512b9f98548de45591a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/84d0e3f7727d4512b9f98548de45591a 2024-11-21T21:26:04,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/84d0e3f7727d4512b9f98548de45591a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T21:26:04,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c74aaba9f7218f5ab8983cd7c853d241 in 851ms, sequenceid=154, compaction requested=true 2024-11-21T21:26:04,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:04,175 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:04,175 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:04,178 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:04,178 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:04,178 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:04,179 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5dfaf3e7713d4a68bd59ffc9988e1b53, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b44b0eedce6140059bb312ebf3b6e436, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8c219e19c59a476e85958fd8db8260c3] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=35.5 K 2024-11-21T21:26:04,179 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:04,179 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:04,179 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:04,179 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/78add8f42ecf442ba77b2b9e94994ce9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/f74b475c27f041a387d220ed8b402674, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d0c47964d306434e99d2df5e8e4a6e4f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=37.9 K 2024-11-21T21:26:04,179 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78add8f42ecf442ba77b2b9e94994ce9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224361104 2024-11-21T21:26:04,180 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfaf3e7713d4a68bd59ffc9988e1b53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224361104 2024-11-21T21:26:04,180 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b44b0eedce6140059bb312ebf3b6e436, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732224362292 2024-11-21T21:26:04,180 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f74b475c27f041a387d220ed8b402674, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732224362292 2024-11-21T21:26:04,181 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c219e19c59a476e85958fd8db8260c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732224362992 2024-11-21T21:26:04,181 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0c47964d306434e99d2df5e8e4a6e4f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732224362992 2024-11-21T21:26:04,188 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:04,189 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:04,189 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2372e6231eee4b24add24629927a181e is 50, key is test_row_0/B:col10/1732224363322/Put/seqid=0 2024-11-21T21:26:04,189 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/7fbd17ee910142ac8ad6d857ac54b7e0 is 50, key is test_row_0/A:col10/1732224363322/Put/seqid=0 2024-11-21T21:26:04,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742319_1495 (size=12493) 2024-11-21T21:26:04,218 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/7fbd17ee910142ac8ad6d857ac54b7e0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7fbd17ee910142ac8ad6d857ac54b7e0 2024-11-21T21:26:04,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T21:26:04,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:04,223 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:26:04,231 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 7fbd17ee910142ac8ad6d857ac54b7e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:04,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:04,231 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224364175; duration=0sec 2024-11-21T21:26:04,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:04,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:04,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:04,233 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:04,233 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:04,233 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:04,233 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c91e7a4f02a142c980b384fb62cb125f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/98bac4cb3e464dd0aa12ca5471b9330e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/84d0e3f7727d4512b9f98548de45591a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=35.5 K 2024-11-21T21:26:04,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742320_1496 (size=12493) 2024-11-21T21:26:04,234 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c91e7a4f02a142c980b384fb62cb125f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732224361104 2024-11-21T21:26:04,234 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98bac4cb3e464dd0aa12ca5471b9330e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732224362292 2024-11-21T21:26:04,235 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84d0e3f7727d4512b9f98548de45591a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732224362992 2024-11-21T21:26:04,246 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2372e6231eee4b24add24629927a181e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2372e6231eee4b24add24629927a181e 2024-11-21T21:26:04,251 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 2372e6231eee4b24add24629927a181e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:04,251 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:04,251 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224364175; duration=0sec 2024-11-21T21:26:04,251 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:04,251 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/74fd2faac2f7407f9176de9758bac780 is 50, key is test_row_0/A:col10/1732224363345/Put/seqid=0 2024-11-21T21:26:04,270 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:04,271 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/cb023736d2654d60abaf2bb115e2b1e5 is 50, key is test_row_0/C:col10/1732224363322/Put/seqid=0 2024-11-21T21:26:04,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742321_1497 (size=12151) 2024-11-21T21:26:04,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742322_1498 (size=12493) 2024-11-21T21:26:04,312 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/cb023736d2654d60abaf2bb115e2b1e5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/cb023736d2654d60abaf2bb115e2b1e5 2024-11-21T21:26:04,319 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into cb023736d2654d60abaf2bb115e2b1e5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:04,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:04,319 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224364175; duration=0sec 2024-11-21T21:26:04,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:04,319 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:26:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:04,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:04,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224424577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224424579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224424588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224424590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,599 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224424600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,605 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224424689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,701 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/74fd2faac2f7407f9176de9758bac780 2024-11-21T21:26:04,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224424701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224424709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/74652dd2a9e0494ea46ee397868e3c8f is 50, key is test_row_0/B:col10/1732224363345/Put/seqid=0 2024-11-21T21:26:04,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742323_1499 (size=12151) 2024-11-21T21:26:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:26:04,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224424910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224424920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:04,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224424920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,130 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/74652dd2a9e0494ea46ee397868e3c8f 2024-11-21T21:26:05,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/ab7dd0ff160343ecac1a0989503edf12 is 50, key is test_row_0/C:col10/1732224363345/Put/seqid=0 2024-11-21T21:26:05,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742324_1500 (size=12151) 2024-11-21T21:26:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224425230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224425230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224425240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,553 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/ab7dd0ff160343ecac1a0989503edf12 2024-11-21T21:26:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/74fd2faac2f7407f9176de9758bac780 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/74fd2faac2f7407f9176de9758bac780 2024-11-21T21:26:05,562 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/74fd2faac2f7407f9176de9758bac780, entries=150, sequenceid=165, filesize=11.9 K 2024-11-21T21:26:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/74652dd2a9e0494ea46ee397868e3c8f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/74652dd2a9e0494ea46ee397868e3c8f 2024-11-21T21:26:05,566 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/74652dd2a9e0494ea46ee397868e3c8f, entries=150, sequenceid=165, filesize=11.9 K 2024-11-21T21:26:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/ab7dd0ff160343ecac1a0989503edf12 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/ab7dd0ff160343ecac1a0989503edf12 2024-11-21T21:26:05,570 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/ab7dd0ff160343ecac1a0989503edf12, entries=150, sequenceid=165, filesize=11.9 K 2024-11-21T21:26:05,573 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c74aaba9f7218f5ab8983cd7c853d241 in 1349ms, sequenceid=165, compaction requested=false 2024-11-21T21:26:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T21:26:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T21:26:05,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T21:26:05,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8110 sec 2024-11-21T21:26:05,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.8160 sec 2024-11-21T21:26:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:05,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T21:26:05,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/37ff0dedfdae4704b7e17169b0d9bbf3 is 50, key is test_row_0/A:col10/1732224364577/Put/seqid=0 2024-11-21T21:26:05,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742325_1501 (size=14541) 2024-11-21T21:26:05,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/37ff0dedfdae4704b7e17169b0d9bbf3 2024-11-21T21:26:05,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/f07c4403b21e4aa7b2e6f955f878cdc8 is 50, key is test_row_0/B:col10/1732224364577/Put/seqid=0 2024-11-21T21:26:05,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224425765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224425771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224425772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742326_1502 (size=12151) 2024-11-21T21:26:05,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/f07c4403b21e4aa7b2e6f955f878cdc8 2024-11-21T21:26:05,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/2242c09da8a04b91960dded8d53fe536 is 50, key is test_row_0/C:col10/1732224364577/Put/seqid=0 2024-11-21T21:26:05,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742327_1503 (size=12151) 2024-11-21T21:26:05,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224425880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224425890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224425890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:26:05,920 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:05,920 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T21:26:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:26:05,922 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:05,922 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:05,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:26:06,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T21:26:06,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:06,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:06,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:06,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224426089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224426099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224426109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/2242c09da8a04b91960dded8d53fe536 2024-11-21T21:26:06,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/37ff0dedfdae4704b7e17169b0d9bbf3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/37ff0dedfdae4704b7e17169b0d9bbf3 2024-11-21T21:26:06,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/37ff0dedfdae4704b7e17169b0d9bbf3, entries=200, sequenceid=194, filesize=14.2 K 2024-11-21T21:26:06,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/f07c4403b21e4aa7b2e6f955f878cdc8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f07c4403b21e4aa7b2e6f955f878cdc8 2024-11-21T21:26:06,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f07c4403b21e4aa7b2e6f955f878cdc8, entries=150, sequenceid=194, filesize=11.9 K 2024-11-21T21:26:06,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/2242c09da8a04b91960dded8d53fe536 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/2242c09da8a04b91960dded8d53fe536 2024-11-21T21:26:06,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T21:26:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:06,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/2242c09da8a04b91960dded8d53fe536, entries=150, sequenceid=194, filesize=11.9 K 2024-11-21T21:26:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:06,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c74aaba9f7218f5ab8983cd7c853d241 in 487ms, sequenceid=194, compaction requested=true 2024-11-21T21:26:06,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:06,229 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:06,230 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:06,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:06,231 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:06,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:06,231 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:06,231 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,231 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,231 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7fbd17ee910142ac8ad6d857ac54b7e0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/74fd2faac2f7407f9176de9758bac780, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/37ff0dedfdae4704b7e17169b0d9bbf3] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=38.3 K 2024-11-21T21:26:06,231 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2372e6231eee4b24add24629927a181e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/74652dd2a9e0494ea46ee397868e3c8f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f07c4403b21e4aa7b2e6f955f878cdc8] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=35.9 K 2024-11-21T21:26:06,231 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fbd17ee910142ac8ad6d857ac54b7e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732224362992 2024-11-21T21:26:06,231 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2372e6231eee4b24add24629927a181e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732224362992 2024-11-21T21:26:06,232 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 74652dd2a9e0494ea46ee397868e3c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732224363340 2024-11-21T21:26:06,232 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74fd2faac2f7407f9176de9758bac780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732224363340 2024-11-21T21:26:06,232 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting f07c4403b21e4aa7b2e6f955f878cdc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732224364577 2024-11-21T21:26:06,232 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37ff0dedfdae4704b7e17169b0d9bbf3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732224364575 2024-11-21T21:26:06,239 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:26:06,240 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/73f38f46d5dc4fb984b71f7da8fe81e1 is 50, key is test_row_0/B:col10/1732224364577/Put/seqid=0 2024-11-21T21:26:06,251 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:06,251 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/a3968c4954534371a5e2812a07a41a58 is 50, key is test_row_0/A:col10/1732224364577/Put/seqid=0 2024-11-21T21:26:06,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742328_1504 (size=12595) 2024-11-21T21:26:06,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742329_1505 (size=12595) 2024-11-21T21:26:06,263 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/73f38f46d5dc4fb984b71f7da8fe81e1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/73f38f46d5dc4fb984b71f7da8fe81e1 2024-11-21T21:26:06,268 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 73f38f46d5dc4fb984b71f7da8fe81e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:06,268 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/a3968c4954534371a5e2812a07a41a58 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/a3968c4954534371a5e2812a07a41a58 2024-11-21T21:26:06,268 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:06,268 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224366230; duration=0sec 2024-11-21T21:26:06,269 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:06,269 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:06,269 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:06,269 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:06,269 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:06,269 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,270 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/cb023736d2654d60abaf2bb115e2b1e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/ab7dd0ff160343ecac1a0989503edf12, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/2242c09da8a04b91960dded8d53fe536] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=35.9 K 2024-11-21T21:26:06,270 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting cb023736d2654d60abaf2bb115e2b1e5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732224362992 2024-11-21T21:26:06,270 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ab7dd0ff160343ecac1a0989503edf12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732224363340 2024-11-21T21:26:06,270 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2242c09da8a04b91960dded8d53fe536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732224364577 2024-11-21T21:26:06,274 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into a3968c4954534371a5e2812a07a41a58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:06,274 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:06,275 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224366229; duration=0sec 2024-11-21T21:26:06,275 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:06,275 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:06,277 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:06,278 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/a4467237036340f28a8e7e926237519c is 50, key is test_row_0/C:col10/1732224364577/Put/seqid=0 2024-11-21T21:26:06,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742330_1506 (size=12595) 2024-11-21T21:26:06,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T21:26:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:06,381 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T21:26:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c6d091254d2843c8ae9f9688fa8b183b is 50, key is test_row_0/A:col10/1732224365771/Put/seqid=0 2024-11-21T21:26:06,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742331_1507 (size=12151) 2024-11-21T21:26:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:06,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224426503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224426503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224426503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:26:06,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224426620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224426620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224426620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,694 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/a4467237036340f28a8e7e926237519c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a4467237036340f28a8e7e926237519c 2024-11-21T21:26:06,697 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into a4467237036340f28a8e7e926237519c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:06,697 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:06,697 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224366230; duration=0sec 2024-11-21T21:26:06,698 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:06,698 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:06,789 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c6d091254d2843c8ae9f9688fa8b183b 2024-11-21T21:26:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/a5908fe64fb6466b98f881b5eb0fc142 is 50, key is test_row_0/B:col10/1732224365771/Put/seqid=0 2024-11-21T21:26:06,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742332_1508 (size=12151) 2024-11-21T21:26:06,822 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/a5908fe64fb6466b98f881b5eb0fc142 2024-11-21T21:26:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/dff4cd31723f4191b54ac60db5d52ab4 is 50, key is test_row_0/C:col10/1732224365771/Put/seqid=0 2024-11-21T21:26:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224426839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224426840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224426840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:06,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742333_1509 (size=12151) 2024-11-21T21:26:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:26:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224427159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224427159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224427159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,272 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/dff4cd31723f4191b54ac60db5d52ab4 2024-11-21T21:26:07,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c6d091254d2843c8ae9f9688fa8b183b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c6d091254d2843c8ae9f9688fa8b183b 2024-11-21T21:26:07,278 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c6d091254d2843c8ae9f9688fa8b183b, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T21:26:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/a5908fe64fb6466b98f881b5eb0fc142 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a5908fe64fb6466b98f881b5eb0fc142 2024-11-21T21:26:07,281 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a5908fe64fb6466b98f881b5eb0fc142, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T21:26:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/dff4cd31723f4191b54ac60db5d52ab4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/dff4cd31723f4191b54ac60db5d52ab4 2024-11-21T21:26:07,285 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/dff4cd31723f4191b54ac60db5d52ab4, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T21:26:07,286 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for c74aaba9f7218f5ab8983cd7c853d241 in 905ms, sequenceid=206, compaction requested=false 2024-11-21T21:26:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T21:26:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T21:26:07,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T21:26:07,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3650 sec 2024-11-21T21:26:07,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3720 sec 2024-11-21T21:26:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:07,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T21:26:07,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/b3bcac8f4be14502ba28d757498a5b36 is 50, key is test_row_0/A:col10/1732224367674/Put/seqid=0 2024-11-21T21:26:07,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742334_1510 (size=16931) 2024-11-21T21:26:07,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/b3bcac8f4be14502ba28d757498a5b36 2024-11-21T21:26:07,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/4ffa21ba16c34da9a802c0579df334e5 is 50, key is test_row_0/B:col10/1732224367674/Put/seqid=0 2024-11-21T21:26:07,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224427686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224427690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224427693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742335_1511 (size=12151) 2024-11-21T21:26:07,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/4ffa21ba16c34da9a802c0579df334e5 2024-11-21T21:26:07,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/5d62396918dd4f9eac4087c9d929b0c3 is 50, key is test_row_0/C:col10/1732224367674/Put/seqid=0 2024-11-21T21:26:07,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742336_1512 (size=12151) 2024-11-21T21:26:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224427800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224427809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224427809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224428010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224428020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224428020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:26:08,070 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:08,070 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-21T21:26:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:26:08,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:08,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:08,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:08,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/5d62396918dd4f9eac4087c9d929b0c3 2024-11-21T21:26:08,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/b3bcac8f4be14502ba28d757498a5b36 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b3bcac8f4be14502ba28d757498a5b36 2024-11-21T21:26:08,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b3bcac8f4be14502ba28d757498a5b36, entries=250, sequenceid=235, filesize=16.5 K 2024-11-21T21:26:08,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/4ffa21ba16c34da9a802c0579df334e5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ffa21ba16c34da9a802c0579df334e5 2024-11-21T21:26:08,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ffa21ba16c34da9a802c0579df334e5, entries=150, sequenceid=235, filesize=11.9 K 2024-11-21T21:26:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/5d62396918dd4f9eac4087c9d929b0c3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/5d62396918dd4f9eac4087c9d929b0c3 2024-11-21T21:26:08,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/5d62396918dd4f9eac4087c9d929b0c3, entries=150, sequenceid=235, filesize=11.9 K 2024-11-21T21:26:08,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for c74aaba9f7218f5ab8983cd7c853d241 in 463ms, sequenceid=235, compaction requested=true 2024-11-21T21:26:08,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:08,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:08,138 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:08,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:08,138 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:08,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:08,139 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:08,139 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:08,139 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,139 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/a3968c4954534371a5e2812a07a41a58, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c6d091254d2843c8ae9f9688fa8b183b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b3bcac8f4be14502ba28d757498a5b36] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=40.7 K 2024-11-21T21:26:08,139 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:08,139 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:08,139 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3968c4954534371a5e2812a07a41a58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732224364577 2024-11-21T21:26:08,139 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,139 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/73f38f46d5dc4fb984b71f7da8fe81e1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a5908fe64fb6466b98f881b5eb0fc142, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ffa21ba16c34da9a802c0579df334e5] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.0 K 2024-11-21T21:26:08,139 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 73f38f46d5dc4fb984b71f7da8fe81e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732224364577 2024-11-21T21:26:08,139 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d091254d2843c8ae9f9688fa8b183b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224365763 2024-11-21T21:26:08,140 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a5908fe64fb6466b98f881b5eb0fc142, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224365763 2024-11-21T21:26:08,140 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bcac8f4be14502ba28d757498a5b36, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732224366462 2024-11-21T21:26:08,140 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ffa21ba16c34da9a802c0579df334e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732224366462 2024-11-21T21:26:08,148 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:08,149 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/8521511ab14e4676ad48bc3d88679ca8 is 50, key is test_row_0/B:col10/1732224367674/Put/seqid=0 2024-11-21T21:26:08,155 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:08,155 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/9f994020a4c346ea808f43d6ed9ee773 is 50, key is test_row_0/A:col10/1732224367674/Put/seqid=0 2024-11-21T21:26:08,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742337_1513 (size=12697) 2024-11-21T21:26:08,164 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/8521511ab14e4676ad48bc3d88679ca8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8521511ab14e4676ad48bc3d88679ca8 2024-11-21T21:26:08,169 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 8521511ab14e4676ad48bc3d88679ca8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:08,169 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:08,169 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224368138; duration=0sec 2024-11-21T21:26:08,169 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:08,169 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:08,169 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:08,170 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:08,170 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:08,170 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,170 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a4467237036340f28a8e7e926237519c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/dff4cd31723f4191b54ac60db5d52ab4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/5d62396918dd4f9eac4087c9d929b0c3] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.0 K 2024-11-21T21:26:08,170 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a4467237036340f28a8e7e926237519c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732224364577 2024-11-21T21:26:08,171 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting dff4cd31723f4191b54ac60db5d52ab4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732224365763 2024-11-21T21:26:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:26:08,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742338_1514 (size=12697) 2024-11-21T21:26:08,188 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d62396918dd4f9eac4087c9d929b0c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732224366462 2024-11-21T21:26:08,197 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/9f994020a4c346ea808f43d6ed9ee773 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9f994020a4c346ea808f43d6ed9ee773 2024-11-21T21:26:08,201 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 9f994020a4c346ea808f43d6ed9ee773(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:08,201 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:08,201 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224368138; duration=0sec 2024-11-21T21:26:08,201 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:08,201 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:08,210 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:08,211 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/959ed116dac845f8b4fb9116e1ba4f83 is 50, key is test_row_0/C:col10/1732224367674/Put/seqid=0 2024-11-21T21:26:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742339_1515 (size=12697) 2024-11-21T21:26:08,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T21:26:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,225 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T21:26:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/65d9d728b5bb4ae3b34f6330de9ba88b is 50, key is test_row_0/A:col10/1732224367691/Put/seqid=0 2024-11-21T21:26:08,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742340_1516 (size=9757) 2024-11-21T21:26:08,235 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/65d9d728b5bb4ae3b34f6330de9ba88b 2024-11-21T21:26:08,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/0491c7fa818748559931b3246798f694 is 50, key is test_row_0/B:col10/1732224367691/Put/seqid=0 2024-11-21T21:26:08,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742341_1517 (size=9757) 2024-11-21T21:26:08,262 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/0491c7fa818748559931b3246798f694 2024-11-21T21:26:08,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/49a9eebe0f7a49a39c2fd682005e4a92 is 50, key is test_row_0/C:col10/1732224367691/Put/seqid=0 2024-11-21T21:26:08,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742342_1518 (size=9757) 2024-11-21T21:26:08,283 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/49a9eebe0f7a49a39c2fd682005e4a92 2024-11-21T21:26:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/65d9d728b5bb4ae3b34f6330de9ba88b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/65d9d728b5bb4ae3b34f6330de9ba88b 2024-11-21T21:26:08,290 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/65d9d728b5bb4ae3b34f6330de9ba88b, entries=100, sequenceid=245, filesize=9.5 K 2024-11-21T21:26:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/0491c7fa818748559931b3246798f694 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/0491c7fa818748559931b3246798f694 2024-11-21T21:26:08,294 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/0491c7fa818748559931b3246798f694, entries=100, sequenceid=245, filesize=9.5 K 2024-11-21T21:26:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/49a9eebe0f7a49a39c2fd682005e4a92 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/49a9eebe0f7a49a39c2fd682005e4a92 2024-11-21T21:26:08,297 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/49a9eebe0f7a49a39c2fd682005e4a92, entries=100, sequenceid=245, filesize=9.5 K 2024-11-21T21:26:08,298 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c74aaba9f7218f5ab8983cd7c853d241 in 73ms, sequenceid=245, compaction requested=false 2024-11-21T21:26:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-21T21:26:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-21T21:26:08,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T21:26:08,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 227 msec 2024-11-21T21:26:08,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 231 msec 2024-11-21T21:26:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:08,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:08,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:26:08,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/691b25e8b460477e95a2ff2b9e08dfc8 is 50, key is test_row_0/A:col10/1732224368333/Put/seqid=0 2024-11-21T21:26:08,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742343_1519 (size=19321) 2024-11-21T21:26:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:26:08,390 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:08,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:08,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224428383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224428390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-21T21:26:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:26:08,399 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:08,400 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:08,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224428398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224428500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224428500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:26:08,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224428510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:08,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:08,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,627 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/959ed116dac845f8b4fb9116e1ba4f83 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/959ed116dac845f8b4fb9116e1ba4f83 2024-11-21T21:26:08,631 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into 959ed116dac845f8b4fb9116e1ba4f83(size=12.4 K), total size for store is 21.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:08,631 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:08,631 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224368139; duration=0sec 2024-11-21T21:26:08,632 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:08,632 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:08,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:08,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:08,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:26:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224428720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224428720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224428720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:08,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/691b25e8b460477e95a2ff2b9e08dfc8 2024-11-21T21:26:08,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/9f3d5ecd44c34d86815b5d4f1305a41e is 50, key is test_row_0/B:col10/1732224368333/Put/seqid=0 2024-11-21T21:26:08,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742344_1520 (size=12151) 2024-11-21T21:26:08,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:08,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:08,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:08,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:26:09,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224429029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:09,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224429029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:09,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224429040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:09,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/9f3d5ecd44c34d86815b5d4f1305a41e 2024-11-21T21:26:09,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/8b95e3c929f044f8be48710da58abbc9 is 50, key is test_row_0/C:col10/1732224368333/Put/seqid=0 2024-11-21T21:26:09,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742345_1521 (size=12151) 2024-11-21T21:26:09,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:09,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:09,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:26:09,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224429550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:09,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224429550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:09,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224429560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:09,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/8b95e3c929f044f8be48710da58abbc9 2024-11-21T21:26:09,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/691b25e8b460477e95a2ff2b9e08dfc8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/691b25e8b460477e95a2ff2b9e08dfc8 2024-11-21T21:26:09,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/691b25e8b460477e95a2ff2b9e08dfc8, entries=300, sequenceid=256, filesize=18.9 K 2024-11-21T21:26:09,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/9f3d5ecd44c34d86815b5d4f1305a41e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/9f3d5ecd44c34d86815b5d4f1305a41e 2024-11-21T21:26:09,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/9f3d5ecd44c34d86815b5d4f1305a41e, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T21:26:09,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/8b95e3c929f044f8be48710da58abbc9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8b95e3c929f044f8be48710da58abbc9 2024-11-21T21:26:09,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8b95e3c929f044f8be48710da58abbc9, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T21:26:09,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c74aaba9f7218f5ab8983cd7c853d241 in 1256ms, sequenceid=256, compaction requested=true 2024-11-21T21:26:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:09,591 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:09,591 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:09,592 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:09,592 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:09,592 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,592 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8521511ab14e4676ad48bc3d88679ca8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/0491c7fa818748559931b3246798f694, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/9f3d5ecd44c34d86815b5d4f1305a41e] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=33.8 K 2024-11-21T21:26:09,592 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41775 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:09,592 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:09,592 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,593 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9f994020a4c346ea808f43d6ed9ee773, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/65d9d728b5bb4ae3b34f6330de9ba88b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/691b25e8b460477e95a2ff2b9e08dfc8] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=40.8 K 2024-11-21T21:26:09,593 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f994020a4c346ea808f43d6ed9ee773, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732224366462 2024-11-21T21:26:09,593 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8521511ab14e4676ad48bc3d88679ca8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732224366462 2024-11-21T21:26:09,593 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65d9d728b5bb4ae3b34f6330de9ba88b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732224367686 2024-11-21T21:26:09,593 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0491c7fa818748559931b3246798f694, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732224367686 2024-11-21T21:26:09,593 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 691b25e8b460477e95a2ff2b9e08dfc8, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224368324 2024-11-21T21:26:09,593 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3d5ecd44c34d86815b5d4f1305a41e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224368324 2024-11-21T21:26:09,602 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:09,603 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/15de381213da47d9ab2fe1d70ccf1036 is 50, key is test_row_0/A:col10/1732224368333/Put/seqid=0 2024-11-21T21:26:09,606 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:09,606 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/aa22ea4cee0d4fc1a717e70ef660b127 is 50, key is test_row_0/B:col10/1732224368333/Put/seqid=0 2024-11-21T21:26:09,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742346_1522 (size=12799) 2024-11-21T21:26:09,617 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/15de381213da47d9ab2fe1d70ccf1036 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/15de381213da47d9ab2fe1d70ccf1036 2024-11-21T21:26:09,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742347_1523 (size=12799) 2024-11-21T21:26:09,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:26:09,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,625 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T21:26:09,626 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 15de381213da47d9ab2fe1d70ccf1036(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:09,627 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:09,627 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224369591; duration=0sec 2024-11-21T21:26:09,627 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:09,627 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:09,627 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:09,628 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/aa22ea4cee0d4fc1a717e70ef660b127 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/aa22ea4cee0d4fc1a717e70ef660b127 2024-11-21T21:26:09,633 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:09,634 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:09,634 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:09,634 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/959ed116dac845f8b4fb9116e1ba4f83, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/49a9eebe0f7a49a39c2fd682005e4a92, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8b95e3c929f044f8be48710da58abbc9] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=33.8 K 2024-11-21T21:26:09,634 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959ed116dac845f8b4fb9116e1ba4f83, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732224366462 2024-11-21T21:26:09,636 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49a9eebe0f7a49a39c2fd682005e4a92, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732224367686 2024-11-21T21:26:09,639 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b95e3c929f044f8be48710da58abbc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224368324 2024-11-21T21:26:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/77cdf982ef794b2189054c3299dc0e38 is 50, key is test_row_0/A:col10/1732224368381/Put/seqid=0 2024-11-21T21:26:09,644 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into aa22ea4cee0d4fc1a717e70ef660b127(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:09,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:09,644 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224369591; duration=0sec 2024-11-21T21:26:09,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:09,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:09,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742348_1524 (size=12301) 2024-11-21T21:26:09,666 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:09,666 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/530c5c3ff54a469e94f4a5dda5be5a85 is 50, key is test_row_0/C:col10/1732224368333/Put/seqid=0 2024-11-21T21:26:09,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742349_1525 (size=12799) 2024-11-21T21:26:10,062 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/77cdf982ef794b2189054c3299dc0e38 2024-11-21T21:26:10,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/50f48471ee0e4957aa54589e0298054f is 50, key is test_row_0/B:col10/1732224368381/Put/seqid=0 2024-11-21T21:26:10,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742350_1526 (size=12301) 2024-11-21T21:26:10,075 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/50f48471ee0e4957aa54589e0298054f 2024-11-21T21:26:10,078 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/530c5c3ff54a469e94f4a5dda5be5a85 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/530c5c3ff54a469e94f4a5dda5be5a85 2024-11-21T21:26:10,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/66e8d185962f47bb9aa6fc3ff70ffed6 is 50, key is test_row_0/C:col10/1732224368381/Put/seqid=0 2024-11-21T21:26:10,085 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into 530c5c3ff54a469e94f4a5dda5be5a85(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:10,085 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:10,085 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224369591; duration=0sec 2024-11-21T21:26:10,086 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:10,086 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:10,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742351_1527 (size=12301) 2024-11-21T21:26:10,530 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/66e8d185962f47bb9aa6fc3ff70ffed6 2024-11-21T21:26:10,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/77cdf982ef794b2189054c3299dc0e38 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/77cdf982ef794b2189054c3299dc0e38 2024-11-21T21:26:10,535 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/77cdf982ef794b2189054c3299dc0e38, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T21:26:10,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/50f48471ee0e4957aa54589e0298054f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/50f48471ee0e4957aa54589e0298054f 2024-11-21T21:26:10,539 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/50f48471ee0e4957aa54589e0298054f, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T21:26:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/66e8d185962f47bb9aa6fc3ff70ffed6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/66e8d185962f47bb9aa6fc3ff70ffed6 2024-11-21T21:26:10,542 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/66e8d185962f47bb9aa6fc3ff70ffed6, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T21:26:10,543 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for c74aaba9f7218f5ab8983cd7c853d241 in 918ms, sequenceid=283, compaction requested=false 2024-11-21T21:26:10,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:10,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:10,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-21T21:26:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-21T21:26:10,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T21:26:10,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1450 sec 2024-11-21T21:26:10,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.1570 sec 2024-11-21T21:26:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:26:10,551 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:10,551 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-21T21:26:10,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:26:10,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:10,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:10,585 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:10,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:26:10,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/3fbb1c29513143afb9c4e1ad92e57d01 is 50, key is test_row_0/A:col10/1732224370584/Put/seqid=0 2024-11-21T21:26:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742352_1528 (size=12301) 2024-11-21T21:26:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:26:10,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224430671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:10,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224430672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224430676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:10,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:10,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224430780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224430790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224430790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:10,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:10,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:26:10,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/3fbb1c29513143afb9c4e1ad92e57d01 2024-11-21T21:26:10,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224430989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224431000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224431000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/5d9670d7298345b7bef558bcc275c16a is 50, key is test_row_0/B:col10/1732224370584/Put/seqid=0 2024-11-21T21:26:11,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742353_1529 (size=12301) 2024-11-21T21:26:11,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:26:11,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224431310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224431310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224431310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/5d9670d7298345b7bef558bcc275c16a 2024-11-21T21:26:11,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/a53c5db34f4449a4b7e6e708f653b6f1 is 50, key is test_row_0/C:col10/1732224370584/Put/seqid=0 2024-11-21T21:26:11,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742354_1530 (size=12301) 2024-11-21T21:26:11,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:11,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:26:11,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224431829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224431829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224431830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:11,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/a53c5db34f4449a4b7e6e708f653b6f1 2024-11-21T21:26:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/3fbb1c29513143afb9c4e1ad92e57d01 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/3fbb1c29513143afb9c4e1ad92e57d01 2024-11-21T21:26:11,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/3fbb1c29513143afb9c4e1ad92e57d01, entries=150, sequenceid=297, filesize=12.0 K 2024-11-21T21:26:11,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/5d9670d7298345b7bef558bcc275c16a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5d9670d7298345b7bef558bcc275c16a 2024-11-21T21:26:11,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5d9670d7298345b7bef558bcc275c16a, entries=150, sequenceid=297, filesize=12.0 K 2024-11-21T21:26:11,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/a53c5db34f4449a4b7e6e708f653b6f1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a53c5db34f4449a4b7e6e708f653b6f1 2024-11-21T21:26:11,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a53c5db34f4449a4b7e6e708f653b6f1, entries=150, sequenceid=297, filesize=12.0 K 2024-11-21T21:26:11,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c74aaba9f7218f5ab8983cd7c853d241 in 1286ms, sequenceid=297, compaction requested=true 2024-11-21T21:26:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:11,871 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:11,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:11,872 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:11,872 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:11,872 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:11,872 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,873 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/15de381213da47d9ab2fe1d70ccf1036, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/77cdf982ef794b2189054c3299dc0e38, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/3fbb1c29513143afb9c4e1ad92e57d01] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.5 K 2024-11-21T21:26:11,874 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:11,874 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:11,874 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,874 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/aa22ea4cee0d4fc1a717e70ef660b127, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/50f48471ee0e4957aa54589e0298054f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5d9670d7298345b7bef558bcc275c16a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.5 K 2024-11-21T21:26:11,874 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15de381213da47d9ab2fe1d70ccf1036, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224368324 2024-11-21T21:26:11,874 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting aa22ea4cee0d4fc1a717e70ef660b127, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224368324 2024-11-21T21:26:11,875 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77cdf982ef794b2189054c3299dc0e38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732224368359 2024-11-21T21:26:11,875 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fbb1c29513143afb9c4e1ad92e57d01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732224370580 2024-11-21T21:26:11,875 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 50f48471ee0e4957aa54589e0298054f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732224368359 2024-11-21T21:26:11,875 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d9670d7298345b7bef558bcc275c16a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732224370580 2024-11-21T21:26:11,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:11,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:11,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:11,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:11,890 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:11,891 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/5bc364662ebc4d2591c66666d6e5706f is 50, key is test_row_0/B:col10/1732224370584/Put/seqid=0 2024-11-21T21:26:11,894 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:11,895 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/7a0cb74fdc594e94a562ed4ce303b7f9 is 50, key is test_row_0/A:col10/1732224370584/Put/seqid=0 2024-11-21T21:26:11,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742356_1532 (size=13051) 2024-11-21T21:26:11,934 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/7a0cb74fdc594e94a562ed4ce303b7f9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7a0cb74fdc594e94a562ed4ce303b7f9 2024-11-21T21:26:11,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:26:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,935 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:26:11,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742355_1531 (size=13051) 2024-11-21T21:26:11,945 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 7a0cb74fdc594e94a562ed4ce303b7f9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:11,946 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:11,946 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224371871; duration=0sec 2024-11-21T21:26:11,946 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:11,946 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:11,946 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:11,949 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:11,949 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:11,950 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:11,950 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/530c5c3ff54a469e94f4a5dda5be5a85, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/66e8d185962f47bb9aa6fc3ff70ffed6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a53c5db34f4449a4b7e6e708f653b6f1] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.5 K 2024-11-21T21:26:11,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/313cab4055a84739898acf2fae3e2bc0 is 50, key is test_row_0/A:col10/1732224370675/Put/seqid=0 2024-11-21T21:26:11,951 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/5bc364662ebc4d2591c66666d6e5706f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5bc364662ebc4d2591c66666d6e5706f 2024-11-21T21:26:11,952 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 530c5c3ff54a469e94f4a5dda5be5a85, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224368324 2024-11-21T21:26:11,955 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66e8d185962f47bb9aa6fc3ff70ffed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732224368359 2024-11-21T21:26:11,956 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a53c5db34f4449a4b7e6e708f653b6f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732224370580 2024-11-21T21:26:11,959 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 5bc364662ebc4d2591c66666d6e5706f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:11,959 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:11,959 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224371872; duration=0sec 2024-11-21T21:26:11,959 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:11,959 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:11,970 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:11,971 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/25acf0869a924917be800fdf242a8e68 is 50, key is test_row_0/C:col10/1732224370584/Put/seqid=0 2024-11-21T21:26:11,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742357_1533 (size=12301) 2024-11-21T21:26:11,981 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/313cab4055a84739898acf2fae3e2bc0 2024-11-21T21:26:11,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/643ebe7ce5bb49a1897cbd06458a7676 is 50, key is test_row_0/B:col10/1732224370675/Put/seqid=0 2024-11-21T21:26:12,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742358_1534 (size=13051) 2024-11-21T21:26:12,012 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/25acf0869a924917be800fdf242a8e68 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/25acf0869a924917be800fdf242a8e68 2024-11-21T21:26:12,022 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into 25acf0869a924917be800fdf242a8e68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:12,022 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:12,022 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224371879; duration=0sec 2024-11-21T21:26:12,023 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:12,023 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:12,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742359_1535 (size=12301) 2024-11-21T21:26:12,426 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/643ebe7ce5bb49a1897cbd06458a7676 2024-11-21T21:26:12,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/14a325a8a1ff411db59a8ed31e78f78f is 50, key is test_row_0/C:col10/1732224370675/Put/seqid=0 2024-11-21T21:26:12,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742360_1536 (size=12301) 2024-11-21T21:26:12,442 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/14a325a8a1ff411db59a8ed31e78f78f 2024-11-21T21:26:12,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/313cab4055a84739898acf2fae3e2bc0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/313cab4055a84739898acf2fae3e2bc0 2024-11-21T21:26:12,456 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/313cab4055a84739898acf2fae3e2bc0, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T21:26:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/643ebe7ce5bb49a1897cbd06458a7676 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/643ebe7ce5bb49a1897cbd06458a7676 2024-11-21T21:26:12,468 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/643ebe7ce5bb49a1897cbd06458a7676, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T21:26:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/14a325a8a1ff411db59a8ed31e78f78f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/14a325a8a1ff411db59a8ed31e78f78f 2024-11-21T21:26:12,475 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/14a325a8a1ff411db59a8ed31e78f78f, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T21:26:12,476 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for c74aaba9f7218f5ab8983cd7c853d241 in 541ms, sequenceid=321, compaction requested=false 2024-11-21T21:26:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-21T21:26:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-21T21:26:12,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-21T21:26:12,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9230 sec 2024-11-21T21:26:12,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.9290 sec 2024-11-21T21:26:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:26:12,700 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:12,701 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-21T21:26:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:26:12,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:12,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:12,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:26:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:12,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:26:12,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:26:12,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:12,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:12,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/61090869cbd348e29757641eb4a7f04c is 50, key is test_row_0/A:col10/1732224372858/Put/seqid=0 2024-11-21T21:26:12,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742361_1537 (size=12301) 2024-11-21T21:26:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224432922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:12,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224432928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:12,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224432933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:12,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:12,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:12,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:26:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:26:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224433040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224433040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224433050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:26:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224433260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224433260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224433260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/61090869cbd348e29757641eb4a7f04c 2024-11-21T21:26:13,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/fe39dd091e14430ab267e495c3cda0e8 is 50, key is test_row_0/B:col10/1732224372858/Put/seqid=0 2024-11-21T21:26:13,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742362_1538 (size=12301) 2024-11-21T21:26:13,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:26:13,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:13,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:26:13,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:26:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224433570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224433570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224433571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:13,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:26:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:13,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/fe39dd091e14430ab267e495c3cda0e8 2024-11-21T21:26:13,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/713b9121dc9c4814a9fe6840695d8c06 is 50, key is test_row_0/C:col10/1732224372858/Put/seqid=0 2024-11-21T21:26:13,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742363_1539 (size=12301) 2024-11-21T21:26:13,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/713b9121dc9c4814a9fe6840695d8c06 2024-11-21T21:26:13,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/61090869cbd348e29757641eb4a7f04c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/61090869cbd348e29757641eb4a7f04c 2024-11-21T21:26:13,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/61090869cbd348e29757641eb4a7f04c, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T21:26:13,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/fe39dd091e14430ab267e495c3cda0e8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/fe39dd091e14430ab267e495c3cda0e8 2024-11-21T21:26:13,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/fe39dd091e14430ab267e495c3cda0e8, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T21:26:13,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/713b9121dc9c4814a9fe6840695d8c06 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/713b9121dc9c4814a9fe6840695d8c06 2024-11-21T21:26:13,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/713b9121dc9c4814a9fe6840695d8c06, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T21:26:13,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c74aaba9f7218f5ab8983cd7c853d241 in 848ms, sequenceid=336, compaction requested=true 2024-11-21T21:26:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:13,708 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:13,708 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:13,709 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:13,709 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:13,709 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:13,709 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:13,709 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,709 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,709 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5bc364662ebc4d2591c66666d6e5706f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/643ebe7ce5bb49a1897cbd06458a7676, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/fe39dd091e14430ab267e495c3cda0e8] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.8 K 2024-11-21T21:26:13,709 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7a0cb74fdc594e94a562ed4ce303b7f9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/313cab4055a84739898acf2fae3e2bc0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/61090869cbd348e29757641eb4a7f04c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.8 K 2024-11-21T21:26:13,709 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0cb74fdc594e94a562ed4ce303b7f9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732224370580 2024-11-21T21:26:13,709 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc364662ebc4d2591c66666d6e5706f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732224370580 2024-11-21T21:26:13,710 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 643ebe7ce5bb49a1897cbd06458a7676, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732224370656 2024-11-21T21:26:13,710 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313cab4055a84739898acf2fae3e2bc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732224370656 2024-11-21T21:26:13,710 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting fe39dd091e14430ab267e495c3cda0e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732224372851 2024-11-21T21:26:13,710 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61090869cbd348e29757641eb4a7f04c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732224372851 2024-11-21T21:26:13,715 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:13,716 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/adb5888ddd7c4dd695be8508b4cd9221 is 50, key is test_row_0/B:col10/1732224372858/Put/seqid=0 2024-11-21T21:26:13,717 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:13,717 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/9dcf71a8ea694474b48f5af3c96ef28d is 50, key is test_row_0/A:col10/1732224372858/Put/seqid=0 2024-11-21T21:26:13,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742364_1540 (size=13153) 2024-11-21T21:26:13,728 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/adb5888ddd7c4dd695be8508b4cd9221 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/adb5888ddd7c4dd695be8508b4cd9221 2024-11-21T21:26:13,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742365_1541 (size=13153) 2024-11-21T21:26:13,732 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into adb5888ddd7c4dd695be8508b4cd9221(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:13,732 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:13,732 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224373708; duration=0sec 2024-11-21T21:26:13,732 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:13,732 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:13,732 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:13,732 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:13,732 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:13,732 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,733 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/25acf0869a924917be800fdf242a8e68, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/14a325a8a1ff411db59a8ed31e78f78f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/713b9121dc9c4814a9fe6840695d8c06] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.8 K 2024-11-21T21:26:13,733 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 25acf0869a924917be800fdf242a8e68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732224370580 2024-11-21T21:26:13,735 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a325a8a1ff411db59a8ed31e78f78f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732224370656 2024-11-21T21:26:13,735 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 713b9121dc9c4814a9fe6840695d8c06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732224372851 2024-11-21T21:26:13,741 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:13,742 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c81443a757074f43b15706bf28e99e5e is 50, key is test_row_0/C:col10/1732224372858/Put/seqid=0 2024-11-21T21:26:13,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742366_1542 (size=13153) 2024-11-21T21:26:13,749 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c81443a757074f43b15706bf28e99e5e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c81443a757074f43b15706bf28e99e5e 2024-11-21T21:26:13,754 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into c81443a757074f43b15706bf28e99e5e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:13,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:13,754 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224373708; duration=0sec 2024-11-21T21:26:13,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:13,754 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:13,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:26:13,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:13,781 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:26:13,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/5b8e26ca6a64445c9d687bebe0ebb923 is 50, key is test_row_0/A:col10/1732224372930/Put/seqid=0 2024-11-21T21:26:13,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742367_1543 (size=12301) 2024-11-21T21:26:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:26:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:14,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. as already flushing 2024-11-21T21:26:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224434106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224434107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224434108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,133 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/9dcf71a8ea694474b48f5af3c96ef28d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9dcf71a8ea694474b48f5af3c96ef28d 2024-11-21T21:26:14,136 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 9dcf71a8ea694474b48f5af3c96ef28d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:14,136 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:14,136 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224373708; duration=0sec 2024-11-21T21:26:14,136 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:14,136 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:14,190 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/5b8e26ca6a64445c9d687bebe0ebb923 2024-11-21T21:26:14,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/46aa3a227f0840e888494345700a7a93 is 50, key is test_row_0/B:col10/1732224372930/Put/seqid=0 2024-11-21T21:26:14,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742368_1544 (size=12301) 2024-11-21T21:26:14,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224434219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224434219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224434219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224434430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224434430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224434430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,598 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/46aa3a227f0840e888494345700a7a93 2024-11-21T21:26:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/354d6f058fd049618bb7a6c26b44ed4e is 50, key is test_row_0/C:col10/1732224372930/Put/seqid=0 2024-11-21T21:26:14,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742369_1545 (size=12301) 2024-11-21T21:26:14,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1732224434659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,665 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1732224434689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,695 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in c74aaba9f7218f5ab8983cd7c853d241 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18321 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224434740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224434740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224434740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:26:15,006 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/354d6f058fd049618bb7a6c26b44ed4e 2024-11-21T21:26:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/5b8e26ca6a64445c9d687bebe0ebb923 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/5b8e26ca6a64445c9d687bebe0ebb923 2024-11-21T21:26:15,014 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/5b8e26ca6a64445c9d687bebe0ebb923, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T21:26:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/46aa3a227f0840e888494345700a7a93 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/46aa3a227f0840e888494345700a7a93 2024-11-21T21:26:15,018 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/46aa3a227f0840e888494345700a7a93, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T21:26:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/354d6f058fd049618bb7a6c26b44ed4e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/354d6f058fd049618bb7a6c26b44ed4e 2024-11-21T21:26:15,022 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/354d6f058fd049618bb7a6c26b44ed4e, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T21:26:15,023 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c74aaba9f7218f5ab8983cd7c853d241 in 1242ms, sequenceid=362, compaction requested=false 2024-11-21T21:26:15,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:15,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:15,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-21T21:26:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-21T21:26:15,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-21T21:26:15,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-11-21T21:26:15,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.3290 sec 2024-11-21T21:26:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:15,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:15,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:26:15,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/be076a4349b348cea4649d75b3d3aa8a is 50, key is test_row_0/A:col10/1732224374104/Put/seqid=0 2024-11-21T21:26:15,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742370_1546 (size=14741) 2024-11-21T21:26:15,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/be076a4349b348cea4649d75b3d3aa8a 2024-11-21T21:26:15,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/7de1b5eb4c194747b2b3786835013991 is 50, key is test_row_0/B:col10/1732224374104/Put/seqid=0 2024-11-21T21:26:15,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742371_1547 (size=12301) 2024-11-21T21:26:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224435326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224435337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224435339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224435440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224435449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224435461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224435650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224435669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224435670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:15,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/7de1b5eb4c194747b2b3786835013991 2024-11-21T21:26:15,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/6959ac0594fc4f38b3669f09429abefe is 50, key is test_row_0/C:col10/1732224374104/Put/seqid=0 2024-11-21T21:26:15,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742372_1548 (size=12301) 2024-11-21T21:26:15,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/6959ac0594fc4f38b3669f09429abefe 2024-11-21T21:26:15,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/be076a4349b348cea4649d75b3d3aa8a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/be076a4349b348cea4649d75b3d3aa8a 2024-11-21T21:26:15,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/be076a4349b348cea4649d75b3d3aa8a, entries=200, sequenceid=377, filesize=14.4 K 2024-11-21T21:26:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/7de1b5eb4c194747b2b3786835013991 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7de1b5eb4c194747b2b3786835013991 2024-11-21T21:26:15,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7de1b5eb4c194747b2b3786835013991, entries=150, sequenceid=377, filesize=12.0 K 2024-11-21T21:26:15,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/6959ac0594fc4f38b3669f09429abefe as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/6959ac0594fc4f38b3669f09429abefe 2024-11-21T21:26:15,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/6959ac0594fc4f38b3669f09429abefe, entries=150, sequenceid=377, filesize=12.0 K 2024-11-21T21:26:15,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for c74aaba9f7218f5ab8983cd7c853d241 in 537ms, sequenceid=377, compaction requested=true 2024-11-21T21:26:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:15,793 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:15,793 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:15,794 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:15,794 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:15,794 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:15,794 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:15,794 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:15,794 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:15,794 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9dcf71a8ea694474b48f5af3c96ef28d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/5b8e26ca6a64445c9d687bebe0ebb923, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/be076a4349b348cea4649d75b3d3aa8a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=39.3 K 2024-11-21T21:26:15,794 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/adb5888ddd7c4dd695be8508b4cd9221, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/46aa3a227f0840e888494345700a7a93, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7de1b5eb4c194747b2b3786835013991] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.9 K 2024-11-21T21:26:15,794 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting adb5888ddd7c4dd695be8508b4cd9221, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732224372851 2024-11-21T21:26:15,794 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dcf71a8ea694474b48f5af3c96ef28d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732224372851 2024-11-21T21:26:15,795 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b8e26ca6a64445c9d687bebe0ebb923, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732224372920 2024-11-21T21:26:15,795 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 46aa3a227f0840e888494345700a7a93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732224372920 2024-11-21T21:26:15,795 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de1b5eb4c194747b2b3786835013991, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732224374104 2024-11-21T21:26:15,795 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting be076a4349b348cea4649d75b3d3aa8a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732224374104 2024-11-21T21:26:15,802 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:15,802 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:15,803 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/075664c72f85442592e639117d24b6e5 is 50, key is test_row_0/B:col10/1732224374104/Put/seqid=0 2024-11-21T21:26:15,803 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/d2a55c27a1ea4b83b282174250a82f0d is 50, key is test_row_0/A:col10/1732224374104/Put/seqid=0 2024-11-21T21:26:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742373_1549 (size=13255) 2024-11-21T21:26:15,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742374_1550 (size=13255) 2024-11-21T21:26:15,887 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/075664c72f85442592e639117d24b6e5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/075664c72f85442592e639117d24b6e5 2024-11-21T21:26:15,900 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 075664c72f85442592e639117d24b6e5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:15,900 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:15,900 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224375793; duration=0sec 2024-11-21T21:26:15,900 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:15,900 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:15,901 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:15,905 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:15,905 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:15,905 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:15,905 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c81443a757074f43b15706bf28e99e5e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/354d6f058fd049618bb7a6c26b44ed4e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/6959ac0594fc4f38b3669f09429abefe] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=36.9 K 2024-11-21T21:26:15,906 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c81443a757074f43b15706bf28e99e5e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732224372851 2024-11-21T21:26:15,908 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 354d6f058fd049618bb7a6c26b44ed4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732224372920 2024-11-21T21:26:15,908 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6959ac0594fc4f38b3669f09429abefe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732224374104 2024-11-21T21:26:15,938 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:15,938 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/a341fc9b3a664777936434eaf515d35c is 50, key is test_row_0/C:col10/1732224374104/Put/seqid=0 2024-11-21T21:26:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:15,973 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:15,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:26:15,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c417270c50b84c618e65383ffc1fdbf2 is 50, key is test_row_0/A:col10/1732224375335/Put/seqid=0 2024-11-21T21:26:15,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742375_1551 (size=13255) 2024-11-21T21:26:16,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742376_1552 (size=14741) 2024-11-21T21:26:16,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c417270c50b84c618e65383ffc1fdbf2 2024-11-21T21:26:16,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2c41bb80f3e74aa49a257dbe717c386c is 50, key is test_row_0/B:col10/1732224375335/Put/seqid=0 2024-11-21T21:26:16,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742377_1553 (size=12301) 2024-11-21T21:26:16,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2c41bb80f3e74aa49a257dbe717c386c 2024-11-21T21:26:16,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c6009914a0da4548ab4694b1b30f4e82 is 50, key is test_row_0/C:col10/1732224375335/Put/seqid=0 2024-11-21T21:26:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224436071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:16,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:16,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224436071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:16,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:16,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742378_1554 (size=12301) 2024-11-21T21:26:16,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224436073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:16,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c6009914a0da4548ab4694b1b30f4e82 2024-11-21T21:26:16,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/c417270c50b84c618e65383ffc1fdbf2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c417270c50b84c618e65383ffc1fdbf2 2024-11-21T21:26:16,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c417270c50b84c618e65383ffc1fdbf2, entries=200, sequenceid=402, filesize=14.4 K 2024-11-21T21:26:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/2c41bb80f3e74aa49a257dbe717c386c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2c41bb80f3e74aa49a257dbe717c386c 2024-11-21T21:26:16,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2c41bb80f3e74aa49a257dbe717c386c, entries=150, sequenceid=402, filesize=12.0 K 2024-11-21T21:26:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/c6009914a0da4548ab4694b1b30f4e82 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c6009914a0da4548ab4694b1b30f4e82 2024-11-21T21:26:16,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c6009914a0da4548ab4694b1b30f4e82, entries=150, sequenceid=402, filesize=12.0 K 2024-11-21T21:26:16,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c74aaba9f7218f5ab8983cd7c853d241 in 162ms, sequenceid=402, compaction requested=false 2024-11-21T21:26:16,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:16,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:16,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T21:26:16,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/1e5f512b6d0247e58222e2602d2beea9 is 50, key is test_row_0/A:col10/1732224376183/Put/seqid=0 2024-11-21T21:26:16,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742379_1555 (size=14741) 2024-11-21T21:26:16,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/1e5f512b6d0247e58222e2602d2beea9 2024-11-21T21:26:16,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/7f4b56a90865467dbb5b495dc4cf9119 is 50, key is test_row_0/B:col10/1732224376183/Put/seqid=0 2024-11-21T21:26:16,278 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/d2a55c27a1ea4b83b282174250a82f0d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d2a55c27a1ea4b83b282174250a82f0d 2024-11-21T21:26:16,283 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into d2a55c27a1ea4b83b282174250a82f0d(size=12.9 K), total size for store is 27.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:16,283 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:16,283 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224375793; duration=0sec 2024-11-21T21:26:16,284 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:16,284 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:16,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742380_1556 (size=12301) 2024-11-21T21:26:16,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/7f4b56a90865467dbb5b495dc4cf9119 2024-11-21T21:26:16,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/337ed302dc954b868c26b63ca7f006ba is 50, key is test_row_0/C:col10/1732224376183/Put/seqid=0 2024-11-21T21:26:16,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1732224436293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:16,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36190 deadline: 1732224436305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:16,317 INFO [Thread-2093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2093. 2024-11-21T21:26:16,317 DEBUG [Thread-2093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,317 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,317 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1732224436307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 , the old value is region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., hostname=849dee24a242,34123,1732224229084, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:26:16,322 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-11-21T21:26:16,322 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,322 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,322 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,325 INFO [Thread-2087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2087. 2024-11-21T21:26:16,325 DEBUG [Thread-2087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,325 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,325 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742381_1557 (size=12301) 2024-11-21T21:26:16,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/337ed302dc954b868c26b63ca7f006ba 2024-11-21T21:26:16,329 INFO [Thread-2089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2089. 2024-11-21T21:26:16,329 DEBUG [Thread-2089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,329 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,329 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/1e5f512b6d0247e58222e2602d2beea9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1e5f512b6d0247e58222e2602d2beea9 2024-11-21T21:26:16,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1e5f512b6d0247e58222e2602d2beea9, entries=200, sequenceid=415, filesize=14.4 K 2024-11-21T21:26:16,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/7f4b56a90865467dbb5b495dc4cf9119 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7f4b56a90865467dbb5b495dc4cf9119 2024-11-21T21:26:16,338 INFO [Thread-2091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2091. 2024-11-21T21:26:16,338 DEBUG [Thread-2091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,338 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,338 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7f4b56a90865467dbb5b495dc4cf9119, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T21:26:16,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/337ed302dc954b868c26b63ca7f006ba as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/337ed302dc954b868c26b63ca7f006ba 2024-11-21T21:26:16,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/337ed302dc954b868c26b63ca7f006ba, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T21:26:16,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c74aaba9f7218f5ab8983cd7c853d241 in 161ms, sequenceid=415, compaction requested=true 2024-11-21T21:26:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:A, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:16,350 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:B, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:16,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:16,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c74aaba9f7218f5ab8983cd7c853d241:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:16,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:26:16,351 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:16,351 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/A is initiating minor compaction (all files) 2024-11-21T21:26:16,351 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/A in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:16,352 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d2a55c27a1ea4b83b282174250a82f0d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c417270c50b84c618e65383ffc1fdbf2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1e5f512b6d0247e58222e2602d2beea9] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=41.7 K 2024-11-21T21:26:16,352 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a55c27a1ea4b83b282174250a82f0d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732224374104 2024-11-21T21:26:16,352 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c417270c50b84c618e65383ffc1fdbf2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732224375324 2024-11-21T21:26:16,352 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5f512b6d0247e58222e2602d2beea9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732224375998 2024-11-21T21:26:16,357 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#A#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:16,358 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/0e0aab22ada34957992a3e3e4d490db5 is 50, key is test_row_0/A:col10/1732224376183/Put/seqid=0 2024-11-21T21:26:16,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742382_1558 (size=13357) 2024-11-21T21:26:16,395 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/a341fc9b3a664777936434eaf515d35c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a341fc9b3a664777936434eaf515d35c 2024-11-21T21:26:16,399 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into a341fc9b3a664777936434eaf515d35c(size=12.9 K), total size for store is 37.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:16,399 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:16,399 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224375793; duration=0sec 2024-11-21T21:26:16,399 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:26:16,399 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:16,399 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:16,399 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:16,401 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:16,401 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/B is initiating minor compaction (all files) 2024-11-21T21:26:16,401 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/B in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:16,401 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/075664c72f85442592e639117d24b6e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2c41bb80f3e74aa49a257dbe717c386c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7f4b56a90865467dbb5b495dc4cf9119] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=37.0 K 2024-11-21T21:26:16,402 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 075664c72f85442592e639117d24b6e5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732224374104 2024-11-21T21:26:16,402 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c41bb80f3e74aa49a257dbe717c386c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732224375335 2024-11-21T21:26:16,402 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f4b56a90865467dbb5b495dc4cf9119, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732224375998 2024-11-21T21:26:16,409 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#B#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:16,410 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/013ab6894117435a9e9887cd944d4798 is 50, key is test_row_0/B:col10/1732224376183/Put/seqid=0 2024-11-21T21:26:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:16,411 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:16,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T21:26:16,412 INFO [Thread-2074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2074. 2024-11-21T21:26:16,412 DEBUG [Thread-2074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,412 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,412 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/07ba7a03a77941d38fd2d62489b2a6cb is 50, key is test_row_0/A:col10/1732224376305/Put/seqid=0 2024-11-21T21:26:16,420 INFO [Thread-2078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2078. 2024-11-21T21:26:16,421 DEBUG [Thread-2078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,421 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,421 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742383_1559 (size=13357) 2024-11-21T21:26:16,429 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/013ab6894117435a9e9887cd944d4798 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/013ab6894117435a9e9887cd944d4798 2024-11-21T21:26:16,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742384_1560 (size=12301) 2024-11-21T21:26:16,432 INFO [Thread-2076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2076. 2024-11-21T21:26:16,432 DEBUG [Thread-2076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:16,432 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,432 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:16,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:16,435 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/B of c74aaba9f7218f5ab8983cd7c853d241 into 013ab6894117435a9e9887cd944d4798(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:16,435 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:16,436 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/B, priority=13, startTime=1732224376350; duration=0sec 2024-11-21T21:26:16,436 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:16,436 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:B 2024-11-21T21:26:16,436 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:16,436 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:16,437 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): c74aaba9f7218f5ab8983cd7c853d241/C is initiating minor compaction (all files) 2024-11-21T21:26:16,437 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c74aaba9f7218f5ab8983cd7c853d241/C in TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:16,437 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a341fc9b3a664777936434eaf515d35c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c6009914a0da4548ab4694b1b30f4e82, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/337ed302dc954b868c26b63ca7f006ba] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp, totalSize=37.0 K 2024-11-21T21:26:16,437 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a341fc9b3a664777936434eaf515d35c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732224374104 2024-11-21T21:26:16,440 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c6009914a0da4548ab4694b1b30f4e82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732224375335 2024-11-21T21:26:16,440 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 337ed302dc954b868c26b63ca7f006ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732224375998 2024-11-21T21:26:16,449 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c74aaba9f7218f5ab8983cd7c853d241#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:16,449 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/4f8c921c7e1f4748958294ff91a3b651 is 50, key is test_row_0/C:col10/1732224376183/Put/seqid=0 2024-11-21T21:26:16,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742385_1561 (size=13357) 2024-11-21T21:26:16,767 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/0e0aab22ada34957992a3e3e4d490db5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0e0aab22ada34957992a3e3e4d490db5 2024-11-21T21:26:16,773 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/A of c74aaba9f7218f5ab8983cd7c853d241 into 0e0aab22ada34957992a3e3e4d490db5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:16,773 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:16,773 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/A, priority=13, startTime=1732224376350; duration=0sec 2024-11-21T21:26:16,773 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:16,773 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:A 2024-11-21T21:26:16,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/07ba7a03a77941d38fd2d62489b2a6cb 2024-11-21T21:26:16,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/f56cc29aa8f84fc8b42ad529f61e2044 is 50, key is test_row_0/B:col10/1732224376305/Put/seqid=0 2024-11-21T21:26:16,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742386_1562 (size=12301) 2024-11-21T21:26:16,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/f56cc29aa8f84fc8b42ad529f61e2044 2024-11-21T21:26:16,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/8c816caf7d8e480e82dc8bc17ac0b5b8 is 50, key is test_row_0/C:col10/1732224376305/Put/seqid=0 2024-11-21T21:26:16,862 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/4f8c921c7e1f4748958294ff91a3b651 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/4f8c921c7e1f4748958294ff91a3b651 2024-11-21T21:26:16,866 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c74aaba9f7218f5ab8983cd7c853d241/C of c74aaba9f7218f5ab8983cd7c853d241 into 4f8c921c7e1f4748958294ff91a3b651(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:16,866 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:16,866 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., storeName=c74aaba9f7218f5ab8983cd7c853d241/C, priority=13, startTime=1732224376351; duration=0sec 2024-11-21T21:26:16,867 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:16,867 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c74aaba9f7218f5ab8983cd7c853d241:C 2024-11-21T21:26:16,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742387_1563 (size=12301) 2024-11-21T21:26:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:26:16,880 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:16,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/8c816caf7d8e480e82dc8bc17ac0b5b8 2024-11-21T21:26:16,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/07ba7a03a77941d38fd2d62489b2a6cb as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/07ba7a03a77941d38fd2d62489b2a6cb 2024-11-21T21:26:16,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/07ba7a03a77941d38fd2d62489b2a6cb, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T21:26:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/f56cc29aa8f84fc8b42ad529f61e2044 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f56cc29aa8f84fc8b42ad529f61e2044 2024-11-21T21:26:16,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f56cc29aa8f84fc8b42ad529f61e2044, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T21:26:16,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/8c816caf7d8e480e82dc8bc17ac0b5b8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8c816caf7d8e480e82dc8bc17ac0b5b8 2024-11-21T21:26:16,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8c816caf7d8e480e82dc8bc17ac0b5b8, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T21:26:16,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=13.42 KB/13740 for c74aaba9f7218f5ab8983cd7c853d241 in 488ms, sequenceid=441, compaction requested=false 2024-11-21T21:26:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c74aaba9f7218f5ab8983cd7c853d241: 2024-11-21T21:26:17,139 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:26:24,711 INFO [Thread-2080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2080. 2024-11-21T21:26:24,711 DEBUG [Thread-2080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:24,711 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:24,711 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:24,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:24,771 INFO [Thread-2082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2082. 2024-11-21T21:26:24,771 DEBUG [Thread-2082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:24,771 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:24,771 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:24,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1444 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4331 rows 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1450 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4349 rows 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1453 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4355 rows 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1449 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4343 rows 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1445 2024-11-21T21:26:24,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4334 rows 2024-11-21T21:26:24,771 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:26:24,771 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:26:24,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:24,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:24,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:24,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:26:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:26:24,775 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224384775"}]},"ts":"1732224384775"} 2024-11-21T21:26:24,776 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:26:24,776 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:26:24,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:26:24,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, UNASSIGN}] 2024-11-21T21:26:24,779 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, UNASSIGN 2024-11-21T21:26:24,779 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c74aaba9f7218f5ab8983cd7c853d241, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:26:24,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, UNASSIGN because future has completed 2024-11-21T21:26:24,781 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:26:24,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:26:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:26:24,933 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:24,933 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:26:24,933 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing c74aaba9f7218f5ab8983cd7c853d241, disabling compactions & flushes 2024-11-21T21:26:24,933 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:24,933 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:24,933 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. after waiting 0 ms 2024-11-21T21:26:24,933 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:24,933 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing c74aaba9f7218f5ab8983cd7c853d241 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-21T21:26:24,936 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/1dc33eb08ebb4038b12b23ec695c7bc6 is 50, key is test_row_0/A:col10/1732224376430/Put/seqid=0 2024-11-21T21:26:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742388_1564 (size=9857) 2024-11-21T21:26:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:26:25,340 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/1dc33eb08ebb4038b12b23ec695c7bc6 2024-11-21T21:26:25,345 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/72ee5f998f8e482bad6918ef7e9417c4 is 50, key is test_row_0/B:col10/1732224376430/Put/seqid=0 2024-11-21T21:26:25,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742389_1565 (size=9857) 2024-11-21T21:26:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:26:25,749 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/72ee5f998f8e482bad6918ef7e9417c4 2024-11-21T21:26:25,753 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/9a2e945a56e4406e951b7d2d95bb7ae1 is 50, key is test_row_0/C:col10/1732224376430/Put/seqid=0 2024-11-21T21:26:25,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742390_1566 (size=9857) 2024-11-21T21:26:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:26:26,156 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/9a2e945a56e4406e951b7d2d95bb7ae1 2024-11-21T21:26:26,160 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/A/1dc33eb08ebb4038b12b23ec695c7bc6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1dc33eb08ebb4038b12b23ec695c7bc6 2024-11-21T21:26:26,163 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1dc33eb08ebb4038b12b23ec695c7bc6, entries=100, sequenceid=451, filesize=9.6 K 2024-11-21T21:26:26,163 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/B/72ee5f998f8e482bad6918ef7e9417c4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/72ee5f998f8e482bad6918ef7e9417c4 2024-11-21T21:26:26,166 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/72ee5f998f8e482bad6918ef7e9417c4, entries=100, sequenceid=451, filesize=9.6 K 2024-11-21T21:26:26,167 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/.tmp/C/9a2e945a56e4406e951b7d2d95bb7ae1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/9a2e945a56e4406e951b7d2d95bb7ae1 2024-11-21T21:26:26,169 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/9a2e945a56e4406e951b7d2d95bb7ae1, entries=100, sequenceid=451, filesize=9.6 K 2024-11-21T21:26:26,170 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c74aaba9f7218f5ab8983cd7c853d241 in 1237ms, sequenceid=451, compaction requested=true 2024-11-21T21:26:26,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/262a383815d940a2bc35581d4c84c2d5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/2bd516886ad54bd1ad7aa863162e3bbf, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0d021bbf24294ca398f123910b40a533, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/99748379a007480ca664056756be79c1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c3fab95373da41ff9cee8109964df3ee, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/fcc5f5b448374bb586b6a556c2f633eb, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b47015b761da42e886c60e6c1c88d973, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/78add8f42ecf442ba77b2b9e94994ce9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/f74b475c27f041a387d220ed8b402674, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d0c47964d306434e99d2df5e8e4a6e4f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7fbd17ee910142ac8ad6d857ac54b7e0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/74fd2faac2f7407f9176de9758bac780, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/37ff0dedfdae4704b7e17169b0d9bbf3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/a3968c4954534371a5e2812a07a41a58, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c6d091254d2843c8ae9f9688fa8b183b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b3bcac8f4be14502ba28d757498a5b36, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9f994020a4c346ea808f43d6ed9ee773, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/65d9d728b5bb4ae3b34f6330de9ba88b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/691b25e8b460477e95a2ff2b9e08dfc8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/15de381213da47d9ab2fe1d70ccf1036, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/77cdf982ef794b2189054c3299dc0e38, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7a0cb74fdc594e94a562ed4ce303b7f9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/3fbb1c29513143afb9c4e1ad92e57d01, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/313cab4055a84739898acf2fae3e2bc0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9dcf71a8ea694474b48f5af3c96ef28d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/61090869cbd348e29757641eb4a7f04c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/5b8e26ca6a64445c9d687bebe0ebb923, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/be076a4349b348cea4649d75b3d3aa8a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d2a55c27a1ea4b83b282174250a82f0d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c417270c50b84c618e65383ffc1fdbf2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1e5f512b6d0247e58222e2602d2beea9] to archive 2024-11-21T21:26:26,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:26:26,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/262a383815d940a2bc35581d4c84c2d5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/262a383815d940a2bc35581d4c84c2d5 2024-11-21T21:26:26,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/2bd516886ad54bd1ad7aa863162e3bbf to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/2bd516886ad54bd1ad7aa863162e3bbf 2024-11-21T21:26:26,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0d021bbf24294ca398f123910b40a533 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0d021bbf24294ca398f123910b40a533 2024-11-21T21:26:26,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/99748379a007480ca664056756be79c1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/99748379a007480ca664056756be79c1 2024-11-21T21:26:26,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c3fab95373da41ff9cee8109964df3ee to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c3fab95373da41ff9cee8109964df3ee 2024-11-21T21:26:26,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/fcc5f5b448374bb586b6a556c2f633eb to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/fcc5f5b448374bb586b6a556c2f633eb 2024-11-21T21:26:26,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b47015b761da42e886c60e6c1c88d973 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b47015b761da42e886c60e6c1c88d973 2024-11-21T21:26:26,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/78add8f42ecf442ba77b2b9e94994ce9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/78add8f42ecf442ba77b2b9e94994ce9 2024-11-21T21:26:26,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/f74b475c27f041a387d220ed8b402674 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/f74b475c27f041a387d220ed8b402674 2024-11-21T21:26:26,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d0c47964d306434e99d2df5e8e4a6e4f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d0c47964d306434e99d2df5e8e4a6e4f 2024-11-21T21:26:26,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7fbd17ee910142ac8ad6d857ac54b7e0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7fbd17ee910142ac8ad6d857ac54b7e0 2024-11-21T21:26:26,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/74fd2faac2f7407f9176de9758bac780 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/74fd2faac2f7407f9176de9758bac780 2024-11-21T21:26:26,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/37ff0dedfdae4704b7e17169b0d9bbf3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/37ff0dedfdae4704b7e17169b0d9bbf3 2024-11-21T21:26:26,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/a3968c4954534371a5e2812a07a41a58 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/a3968c4954534371a5e2812a07a41a58 2024-11-21T21:26:26,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c6d091254d2843c8ae9f9688fa8b183b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c6d091254d2843c8ae9f9688fa8b183b 2024-11-21T21:26:26,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b3bcac8f4be14502ba28d757498a5b36 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/b3bcac8f4be14502ba28d757498a5b36 2024-11-21T21:26:26,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9f994020a4c346ea808f43d6ed9ee773 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9f994020a4c346ea808f43d6ed9ee773 2024-11-21T21:26:26,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/65d9d728b5bb4ae3b34f6330de9ba88b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/65d9d728b5bb4ae3b34f6330de9ba88b 2024-11-21T21:26:26,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/691b25e8b460477e95a2ff2b9e08dfc8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/691b25e8b460477e95a2ff2b9e08dfc8 2024-11-21T21:26:26,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/15de381213da47d9ab2fe1d70ccf1036 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/15de381213da47d9ab2fe1d70ccf1036 2024-11-21T21:26:26,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/77cdf982ef794b2189054c3299dc0e38 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/77cdf982ef794b2189054c3299dc0e38 2024-11-21T21:26:26,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7a0cb74fdc594e94a562ed4ce303b7f9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/7a0cb74fdc594e94a562ed4ce303b7f9 2024-11-21T21:26:26,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/3fbb1c29513143afb9c4e1ad92e57d01 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/3fbb1c29513143afb9c4e1ad92e57d01 2024-11-21T21:26:26,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/313cab4055a84739898acf2fae3e2bc0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/313cab4055a84739898acf2fae3e2bc0 2024-11-21T21:26:26,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9dcf71a8ea694474b48f5af3c96ef28d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/9dcf71a8ea694474b48f5af3c96ef28d 2024-11-21T21:26:26,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/61090869cbd348e29757641eb4a7f04c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/61090869cbd348e29757641eb4a7f04c 2024-11-21T21:26:26,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/5b8e26ca6a64445c9d687bebe0ebb923 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/5b8e26ca6a64445c9d687bebe0ebb923 2024-11-21T21:26:26,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/be076a4349b348cea4649d75b3d3aa8a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/be076a4349b348cea4649d75b3d3aa8a 2024-11-21T21:26:26,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d2a55c27a1ea4b83b282174250a82f0d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/d2a55c27a1ea4b83b282174250a82f0d 2024-11-21T21:26:26,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c417270c50b84c618e65383ffc1fdbf2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/c417270c50b84c618e65383ffc1fdbf2 2024-11-21T21:26:26,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1e5f512b6d0247e58222e2602d2beea9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1e5f512b6d0247e58222e2602d2beea9 2024-11-21T21:26:26,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ce1d3328f8b42e781d88dbe6cdb7c5a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b90cbdff6dc544bf8ba5b4ae8ce691ac, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/59f10e1a8f3b445ab4bdb2146e6c1f97, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2826b9c9574e4cc68b41a42102f0ebba, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/eaa2273b764c441f8448f3e26f7eba68, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/868c147f4c404c438412084cffc1b3c2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5dfaf3e7713d4a68bd59ffc9988e1b53, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a2e44eb31e1746ffb3b5f57d92ce5632, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b44b0eedce6140059bb312ebf3b6e436, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2372e6231eee4b24add24629927a181e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8c219e19c59a476e85958fd8db8260c3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/74652dd2a9e0494ea46ee397868e3c8f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/73f38f46d5dc4fb984b71f7da8fe81e1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f07c4403b21e4aa7b2e6f955f878cdc8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a5908fe64fb6466b98f881b5eb0fc142, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8521511ab14e4676ad48bc3d88679ca8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ffa21ba16c34da9a802c0579df334e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/0491c7fa818748559931b3246798f694, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/aa22ea4cee0d4fc1a717e70ef660b127, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/9f3d5ecd44c34d86815b5d4f1305a41e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/50f48471ee0e4957aa54589e0298054f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5bc364662ebc4d2591c66666d6e5706f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5d9670d7298345b7bef558bcc275c16a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/643ebe7ce5bb49a1897cbd06458a7676, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/adb5888ddd7c4dd695be8508b4cd9221, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/fe39dd091e14430ab267e495c3cda0e8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/46aa3a227f0840e888494345700a7a93, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/075664c72f85442592e639117d24b6e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7de1b5eb4c194747b2b3786835013991, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2c41bb80f3e74aa49a257dbe717c386c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7f4b56a90865467dbb5b495dc4cf9119] to archive 2024-11-21T21:26:26,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:26:26,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ce1d3328f8b42e781d88dbe6cdb7c5a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ce1d3328f8b42e781d88dbe6cdb7c5a 2024-11-21T21:26:26,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b90cbdff6dc544bf8ba5b4ae8ce691ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b90cbdff6dc544bf8ba5b4ae8ce691ac 2024-11-21T21:26:26,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/59f10e1a8f3b445ab4bdb2146e6c1f97 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/59f10e1a8f3b445ab4bdb2146e6c1f97 2024-11-21T21:26:26,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2826b9c9574e4cc68b41a42102f0ebba to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2826b9c9574e4cc68b41a42102f0ebba 2024-11-21T21:26:26,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/eaa2273b764c441f8448f3e26f7eba68 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/eaa2273b764c441f8448f3e26f7eba68 2024-11-21T21:26:26,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/868c147f4c404c438412084cffc1b3c2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/868c147f4c404c438412084cffc1b3c2 2024-11-21T21:26:26,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5dfaf3e7713d4a68bd59ffc9988e1b53 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5dfaf3e7713d4a68bd59ffc9988e1b53 2024-11-21T21:26:26,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a2e44eb31e1746ffb3b5f57d92ce5632 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a2e44eb31e1746ffb3b5f57d92ce5632 2024-11-21T21:26:26,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b44b0eedce6140059bb312ebf3b6e436 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/b44b0eedce6140059bb312ebf3b6e436 2024-11-21T21:26:26,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2372e6231eee4b24add24629927a181e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2372e6231eee4b24add24629927a181e 2024-11-21T21:26:26,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8c219e19c59a476e85958fd8db8260c3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8c219e19c59a476e85958fd8db8260c3 2024-11-21T21:26:26,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/74652dd2a9e0494ea46ee397868e3c8f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/74652dd2a9e0494ea46ee397868e3c8f 2024-11-21T21:26:26,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/73f38f46d5dc4fb984b71f7da8fe81e1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/73f38f46d5dc4fb984b71f7da8fe81e1 2024-11-21T21:26:26,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f07c4403b21e4aa7b2e6f955f878cdc8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f07c4403b21e4aa7b2e6f955f878cdc8 2024-11-21T21:26:26,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a5908fe64fb6466b98f881b5eb0fc142 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/a5908fe64fb6466b98f881b5eb0fc142 2024-11-21T21:26:26,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8521511ab14e4676ad48bc3d88679ca8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/8521511ab14e4676ad48bc3d88679ca8 2024-11-21T21:26:26,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ffa21ba16c34da9a802c0579df334e5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/4ffa21ba16c34da9a802c0579df334e5 2024-11-21T21:26:26,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/0491c7fa818748559931b3246798f694 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/0491c7fa818748559931b3246798f694 2024-11-21T21:26:26,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/aa22ea4cee0d4fc1a717e70ef660b127 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/aa22ea4cee0d4fc1a717e70ef660b127 2024-11-21T21:26:26,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/9f3d5ecd44c34d86815b5d4f1305a41e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/9f3d5ecd44c34d86815b5d4f1305a41e 2024-11-21T21:26:26,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/50f48471ee0e4957aa54589e0298054f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/50f48471ee0e4957aa54589e0298054f 2024-11-21T21:26:26,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5bc364662ebc4d2591c66666d6e5706f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5bc364662ebc4d2591c66666d6e5706f 2024-11-21T21:26:26,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5d9670d7298345b7bef558bcc275c16a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/5d9670d7298345b7bef558bcc275c16a 2024-11-21T21:26:26,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/643ebe7ce5bb49a1897cbd06458a7676 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/643ebe7ce5bb49a1897cbd06458a7676 2024-11-21T21:26:26,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/adb5888ddd7c4dd695be8508b4cd9221 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/adb5888ddd7c4dd695be8508b4cd9221 2024-11-21T21:26:26,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/fe39dd091e14430ab267e495c3cda0e8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/fe39dd091e14430ab267e495c3cda0e8 2024-11-21T21:26:26,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/46aa3a227f0840e888494345700a7a93 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/46aa3a227f0840e888494345700a7a93 2024-11-21T21:26:26,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/075664c72f85442592e639117d24b6e5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/075664c72f85442592e639117d24b6e5 2024-11-21T21:26:26,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7de1b5eb4c194747b2b3786835013991 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7de1b5eb4c194747b2b3786835013991 2024-11-21T21:26:26,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2c41bb80f3e74aa49a257dbe717c386c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/2c41bb80f3e74aa49a257dbe717c386c 2024-11-21T21:26:26,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7f4b56a90865467dbb5b495dc4cf9119 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/7f4b56a90865467dbb5b495dc4cf9119 2024-11-21T21:26:26,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/401f381aae03429289d5ddcdc29624a4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/be64eda2de384a95a8c45d47b7f14cfa, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/26a44ba830dd4f57b788dee96b9728c8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c614e602a04d41a4b501e5b0482c9a71, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/64d27833434a40099a1c529dee6ced4c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/69dab6f58c9b491c8dccf5db73d58bb0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c91e7a4f02a142c980b384fb62cb125f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/1690a4d6ad22451aa6ea1d7bf4108514, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/98bac4cb3e464dd0aa12ca5471b9330e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/cb023736d2654d60abaf2bb115e2b1e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/84d0e3f7727d4512b9f98548de45591a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/ab7dd0ff160343ecac1a0989503edf12, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a4467237036340f28a8e7e926237519c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/2242c09da8a04b91960dded8d53fe536, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/dff4cd31723f4191b54ac60db5d52ab4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/959ed116dac845f8b4fb9116e1ba4f83, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/5d62396918dd4f9eac4087c9d929b0c3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/49a9eebe0f7a49a39c2fd682005e4a92, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/530c5c3ff54a469e94f4a5dda5be5a85, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8b95e3c929f044f8be48710da58abbc9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/66e8d185962f47bb9aa6fc3ff70ffed6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/25acf0869a924917be800fdf242a8e68, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a53c5db34f4449a4b7e6e708f653b6f1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/14a325a8a1ff411db59a8ed31e78f78f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c81443a757074f43b15706bf28e99e5e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/713b9121dc9c4814a9fe6840695d8c06, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/354d6f058fd049618bb7a6c26b44ed4e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a341fc9b3a664777936434eaf515d35c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/6959ac0594fc4f38b3669f09429abefe, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c6009914a0da4548ab4694b1b30f4e82, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/337ed302dc954b868c26b63ca7f006ba] to archive 2024-11-21T21:26:26,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:26:26,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/401f381aae03429289d5ddcdc29624a4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/401f381aae03429289d5ddcdc29624a4 2024-11-21T21:26:26,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/be64eda2de384a95a8c45d47b7f14cfa to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/be64eda2de384a95a8c45d47b7f14cfa 2024-11-21T21:26:26,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/26a44ba830dd4f57b788dee96b9728c8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/26a44ba830dd4f57b788dee96b9728c8 2024-11-21T21:26:26,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c614e602a04d41a4b501e5b0482c9a71 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c614e602a04d41a4b501e5b0482c9a71 2024-11-21T21:26:26,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/64d27833434a40099a1c529dee6ced4c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/64d27833434a40099a1c529dee6ced4c 2024-11-21T21:26:26,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/69dab6f58c9b491c8dccf5db73d58bb0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/69dab6f58c9b491c8dccf5db73d58bb0 2024-11-21T21:26:26,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c91e7a4f02a142c980b384fb62cb125f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c91e7a4f02a142c980b384fb62cb125f 2024-11-21T21:26:26,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/1690a4d6ad22451aa6ea1d7bf4108514 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/1690a4d6ad22451aa6ea1d7bf4108514 2024-11-21T21:26:26,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/98bac4cb3e464dd0aa12ca5471b9330e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/98bac4cb3e464dd0aa12ca5471b9330e 2024-11-21T21:26:26,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/cb023736d2654d60abaf2bb115e2b1e5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/cb023736d2654d60abaf2bb115e2b1e5 2024-11-21T21:26:26,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/84d0e3f7727d4512b9f98548de45591a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/84d0e3f7727d4512b9f98548de45591a 2024-11-21T21:26:26,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/ab7dd0ff160343ecac1a0989503edf12 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/ab7dd0ff160343ecac1a0989503edf12 2024-11-21T21:26:26,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a4467237036340f28a8e7e926237519c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a4467237036340f28a8e7e926237519c 2024-11-21T21:26:26,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/2242c09da8a04b91960dded8d53fe536 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/2242c09da8a04b91960dded8d53fe536 2024-11-21T21:26:26,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/dff4cd31723f4191b54ac60db5d52ab4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/dff4cd31723f4191b54ac60db5d52ab4 2024-11-21T21:26:26,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/959ed116dac845f8b4fb9116e1ba4f83 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/959ed116dac845f8b4fb9116e1ba4f83 2024-11-21T21:26:26,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/5d62396918dd4f9eac4087c9d929b0c3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/5d62396918dd4f9eac4087c9d929b0c3 2024-11-21T21:26:26,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/49a9eebe0f7a49a39c2fd682005e4a92 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/49a9eebe0f7a49a39c2fd682005e4a92 2024-11-21T21:26:26,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/530c5c3ff54a469e94f4a5dda5be5a85 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/530c5c3ff54a469e94f4a5dda5be5a85 2024-11-21T21:26:26,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8b95e3c929f044f8be48710da58abbc9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8b95e3c929f044f8be48710da58abbc9 2024-11-21T21:26:26,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/66e8d185962f47bb9aa6fc3ff70ffed6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/66e8d185962f47bb9aa6fc3ff70ffed6 2024-11-21T21:26:26,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/25acf0869a924917be800fdf242a8e68 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/25acf0869a924917be800fdf242a8e68 2024-11-21T21:26:26,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a53c5db34f4449a4b7e6e708f653b6f1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a53c5db34f4449a4b7e6e708f653b6f1 2024-11-21T21:26:26,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/14a325a8a1ff411db59a8ed31e78f78f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/14a325a8a1ff411db59a8ed31e78f78f 2024-11-21T21:26:26,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c81443a757074f43b15706bf28e99e5e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c81443a757074f43b15706bf28e99e5e 2024-11-21T21:26:26,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/713b9121dc9c4814a9fe6840695d8c06 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/713b9121dc9c4814a9fe6840695d8c06 2024-11-21T21:26:26,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/354d6f058fd049618bb7a6c26b44ed4e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/354d6f058fd049618bb7a6c26b44ed4e 2024-11-21T21:26:26,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a341fc9b3a664777936434eaf515d35c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/a341fc9b3a664777936434eaf515d35c 2024-11-21T21:26:26,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/6959ac0594fc4f38b3669f09429abefe to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/6959ac0594fc4f38b3669f09429abefe 2024-11-21T21:26:26,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c6009914a0da4548ab4694b1b30f4e82 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/c6009914a0da4548ab4694b1b30f4e82 2024-11-21T21:26:26,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/337ed302dc954b868c26b63ca7f006ba to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/337ed302dc954b868c26b63ca7f006ba 2024-11-21T21:26:26,249 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/recovered.edits/454.seqid, newMaxSeqId=454, maxSeqId=1 2024-11-21T21:26:26,250 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. 2024-11-21T21:26:26,250 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for c74aaba9f7218f5ab8983cd7c853d241: Waiting for close lock at 1732224384933Running coprocessor pre-close hooks at 1732224384933Disabling compacts and flushes for region at 1732224384933Disabling writes for close at 1732224384933Obtaining lock to block concurrent updates at 1732224384933Preparing flush snapshotting stores in c74aaba9f7218f5ab8983cd7c853d241 at 1732224384933Finished memstore snapshotting TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732224384933Flushing stores of TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241. at 1732224384934 (+1 ms)Flushing c74aaba9f7218f5ab8983cd7c853d241/A: creating writer at 1732224384934Flushing c74aaba9f7218f5ab8983cd7c853d241/A: appending metadata at 1732224384936 (+2 ms)Flushing c74aaba9f7218f5ab8983cd7c853d241/A: closing flushed file at 1732224384936Flushing c74aaba9f7218f5ab8983cd7c853d241/B: creating writer at 1732224385343 (+407 ms)Flushing c74aaba9f7218f5ab8983cd7c853d241/B: appending metadata at 1732224385345 (+2 ms)Flushing c74aaba9f7218f5ab8983cd7c853d241/B: closing flushed file at 1732224385345Flushing c74aaba9f7218f5ab8983cd7c853d241/C: creating writer at 1732224385751 (+406 ms)Flushing c74aaba9f7218f5ab8983cd7c853d241/C: appending metadata at 1732224385753 (+2 ms)Flushing c74aaba9f7218f5ab8983cd7c853d241/C: closing flushed file at 1732224385753Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@452d521e: reopening flushed file at 1732224386159 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39d5c4b9: reopening flushed file at 1732224386163 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5155afc7: reopening flushed file at 1732224386166 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c74aaba9f7218f5ab8983cd7c853d241 in 1237ms, sequenceid=451, compaction requested=true at 1732224386170 (+4 ms)Writing region close event to WAL at 1732224386247 (+77 ms)Running coprocessor post-close hooks at 1732224386250 (+3 ms)Closed at 1732224386250 2024-11-21T21:26:26,252 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:26,252 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c74aaba9f7218f5ab8983cd7c853d241, regionState=CLOSED 2024-11-21T21:26:26,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:26:26,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T21:26:26,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure c74aaba9f7218f5ab8983cd7c853d241, server=849dee24a242,34123,1732224229084 in 1.4730 sec 2024-11-21T21:26:26,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-21T21:26:26,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c74aaba9f7218f5ab8983cd7c853d241, UNASSIGN in 1.4770 sec 2024-11-21T21:26:26,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T21:26:26,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-21T21:26:26,259 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224386259"}]},"ts":"1732224386259"} 2024-11-21T21:26:26,260 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:26:26,260 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:26:26,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4880 sec 2024-11-21T21:26:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:26:26,919 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:26,920 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:26:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:26,922 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:26:26,923 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:26,924 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:26,926 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/recovered.edits] 2024-11-21T21:26:26,927 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/07ba7a03a77941d38fd2d62489b2a6cb to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/07ba7a03a77941d38fd2d62489b2a6cb 2024-11-21T21:26:26,928 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0e0aab22ada34957992a3e3e4d490db5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/0e0aab22ada34957992a3e3e4d490db5 2024-11-21T21:26:26,929 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1dc33eb08ebb4038b12b23ec695c7bc6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/A/1dc33eb08ebb4038b12b23ec695c7bc6 2024-11-21T21:26:26,930 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/013ab6894117435a9e9887cd944d4798 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/013ab6894117435a9e9887cd944d4798 2024-11-21T21:26:26,931 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/72ee5f998f8e482bad6918ef7e9417c4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/72ee5f998f8e482bad6918ef7e9417c4 2024-11-21T21:26:26,932 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f56cc29aa8f84fc8b42ad529f61e2044 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/B/f56cc29aa8f84fc8b42ad529f61e2044 2024-11-21T21:26:26,933 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/4f8c921c7e1f4748958294ff91a3b651 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/4f8c921c7e1f4748958294ff91a3b651 2024-11-21T21:26:26,934 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8c816caf7d8e480e82dc8bc17ac0b5b8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/8c816caf7d8e480e82dc8bc17ac0b5b8 2024-11-21T21:26:26,935 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/9a2e945a56e4406e951b7d2d95bb7ae1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/C/9a2e945a56e4406e951b7d2d95bb7ae1 2024-11-21T21:26:26,936 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/recovered.edits/454.seqid to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241/recovered.edits/454.seqid 2024-11-21T21:26:26,937 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/c74aaba9f7218f5ab8983cd7c853d241 2024-11-21T21:26:26,937 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:26:26,938 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:26,940 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:26:26,941 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:26:26,942 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:26,942 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:26:26,943 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732224386942"}]},"ts":"9223372036854775807"} 2024-11-21T21:26:26,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:26:26,944 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c74aaba9f7218f5ab8983cd7c853d241, NAME => 'TestAcidGuarantees,,1732224354113.c74aaba9f7218f5ab8983cd7c853d241.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:26:26,944 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:26:26,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732224386944"}]},"ts":"9223372036854775807"} 2024-11-21T21:26:26,946 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:26:26,947 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:26,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-21T21:26:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:26:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:26:27,030 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:27,037 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 242), OpenFileDescriptor=443 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=500 (was 489) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11476 (was 11621) 2024-11-21T21:26:27,045 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=500, ProcessCount=11, AvailableMemoryMB=11476 2024-11-21T21:26:27,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:26:27,047 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:26:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:27,049 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:26:27,049 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:27,049 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-21T21:26:27,049 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:26:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T21:26:27,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742391_1567 (size=959) 2024-11-21T21:26:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T21:26:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T21:26:27,455 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b 2024-11-21T21:26:27,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742392_1568 (size=53) 2024-11-21T21:26:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T21:26:27,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:26:27,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2ddd2a6cb469a423cd30254b88184ba0, disabling compactions & flushes 2024-11-21T21:26:27,860 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:27,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:27,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. after waiting 0 ms 2024-11-21T21:26:27,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:27,860 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:27,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2ddd2a6cb469a423cd30254b88184ba0: Waiting for close lock at 1732224387860Disabling compacts and flushes for region at 1732224387860Disabling writes for close at 1732224387860Writing region close event to WAL at 1732224387860Closed at 1732224387860 2024-11-21T21:26:27,861 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:26:27,862 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732224387861"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732224387861"}]},"ts":"1732224387861"} 2024-11-21T21:26:27,863 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:26:27,863 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:26:27,864 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224387863"}]},"ts":"1732224387863"} 2024-11-21T21:26:27,864 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:26:27,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, ASSIGN}] 2024-11-21T21:26:27,865 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, ASSIGN 2024-11-21T21:26:27,866 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, ASSIGN; state=OFFLINE, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=false 2024-11-21T21:26:28,017 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:26:28,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, ASSIGN because future has completed 2024-11-21T21:26:28,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:26:28,172 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:28,172 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:26:28,173 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,173 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:26:28,173 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,173 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,174 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,175 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ddd2a6cb469a423cd30254b88184ba0 columnFamilyName A 2024-11-21T21:26:28,175 DEBUG [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:28,175 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(327): Store=2ddd2a6cb469a423cd30254b88184ba0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:26:28,175 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,176 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ddd2a6cb469a423cd30254b88184ba0 columnFamilyName B 2024-11-21T21:26:28,176 DEBUG [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:28,177 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(327): Store=2ddd2a6cb469a423cd30254b88184ba0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:26:28,177 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,178 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ddd2a6cb469a423cd30254b88184ba0 columnFamilyName C 2024-11-21T21:26:28,178 DEBUG [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:28,178 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(327): Store=2ddd2a6cb469a423cd30254b88184ba0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:26:28,178 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,179 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,179 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,180 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,180 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,181 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:26:28,182 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,183 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:26:28,184 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 2ddd2a6cb469a423cd30254b88184ba0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63989803, jitterRate=-0.046477630734443665}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:26:28,184 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:28,184 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 2ddd2a6cb469a423cd30254b88184ba0: Running coprocessor pre-open hook at 1732224388173Writing region info on filesystem at 1732224388173Initializing all the Stores at 1732224388173Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224388174 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224388174Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224388174Cleaning up temporary data from old regions at 1732224388180 (+6 ms)Running coprocessor post-open hooks at 1732224388184 (+4 ms)Region opened successfully at 1732224388184 2024-11-21T21:26:28,185 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., pid=148, masterSystemTime=1732224388170 2024-11-21T21:26:28,187 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:28,187 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:28,187 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:26:28,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:26:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T21:26:28,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-21T21:26:28,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 in 171 msec 2024-11-21T21:26:28,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-21T21:26:28,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, ASSIGN in 325 msec 2024-11-21T21:26:28,192 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:26:28,193 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224388192"}]},"ts":"1732224388192"} 2024-11-21T21:26:28,194 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:26:28,195 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:26:28,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-21T21:26:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T21:26:29,210 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:29,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:29,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:29,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:29,212 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8e031c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:29,213 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac8b84e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:29,214 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:29,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:29,216 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:26:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,35047,1732224227985 2024-11-21T21:26:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eaae4a9 2024-11-21T21:26:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:26:29,218 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:26:29,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:26:29,219 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:26:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:29,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742393_1569 (size=995) 2024-11-21T21:26:29,629 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T21:26:29,629 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T21:26:29,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:26:29,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, REOPEN/MOVE}] 2024-11-21T21:26:29,634 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, REOPEN/MOVE 2024-11-21T21:26:29,634 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:26:29,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, REOPEN/MOVE because future has completed 2024-11-21T21:26:29,636 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:26:29,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:26:29,788 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:29,788 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:26:29,788 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 2ddd2a6cb469a423cd30254b88184ba0, disabling compactions & flushes 2024-11-21T21:26:29,788 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:29,789 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:29,789 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. after waiting 0 ms 2024-11-21T21:26:29,789 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:29,792 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T21:26:29,793 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:29,793 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 2ddd2a6cb469a423cd30254b88184ba0: Waiting for close lock at 1732224389788Running coprocessor pre-close hooks at 1732224389788Disabling compacts and flushes for region at 1732224389788Disabling writes for close at 1732224389789 (+1 ms)Writing region close event to WAL at 1732224389789Running coprocessor post-close hooks at 1732224389793 (+4 ms)Closed at 1732224389793 2024-11-21T21:26:29,793 WARN [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 2ddd2a6cb469a423cd30254b88184ba0 to self. 2024-11-21T21:26:29,795 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:29,795 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=CLOSED 2024-11-21T21:26:29,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:26:29,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-21T21:26:29,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 in 161 msec 2024-11-21T21:26:29,799 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, REOPEN/MOVE; state=CLOSED, location=849dee24a242,34123,1732224229084; forceNewPlan=false, retain=true 2024-11-21T21:26:29,949 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=OPENING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:26:29,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, REOPEN/MOVE because future has completed 2024-11-21T21:26:29,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:26:30,105 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,105 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:26:30,106 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,106 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:26:30,106 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,106 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,107 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,108 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ddd2a6cb469a423cd30254b88184ba0 columnFamilyName A 2024-11-21T21:26:30,109 DEBUG [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:30,109 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(327): Store=2ddd2a6cb469a423cd30254b88184ba0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:26:30,109 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,110 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ddd2a6cb469a423cd30254b88184ba0 columnFamilyName B 2024-11-21T21:26:30,110 DEBUG [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:30,110 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(327): Store=2ddd2a6cb469a423cd30254b88184ba0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:26:30,110 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,111 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ddd2a6cb469a423cd30254b88184ba0 columnFamilyName C 2024-11-21T21:26:30,111 DEBUG [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:30,111 INFO [StoreOpener-2ddd2a6cb469a423cd30254b88184ba0-1 {}] regionserver.HStore(327): Store=2ddd2a6cb469a423cd30254b88184ba0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:26:30,112 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,112 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,113 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,114 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,114 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,114 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:26:30,115 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,115 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 2ddd2a6cb469a423cd30254b88184ba0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73485753, jitterRate=0.09502305090427399}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:26:30,116 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,116 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 2ddd2a6cb469a423cd30254b88184ba0: Running coprocessor pre-open hook at 1732224390106Writing region info on filesystem at 1732224390106Initializing all the Stores at 1732224390107 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224390107Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224390107Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732224390107Cleaning up temporary data from old regions at 1732224390114 (+7 ms)Running coprocessor post-open hooks at 1732224390116 (+2 ms)Region opened successfully at 1732224390116 2024-11-21T21:26:30,117 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., pid=153, masterSystemTime=1732224390103 2024-11-21T21:26:30,118 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,118 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,119 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=OPEN, openSeqNum=5, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:26:30,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-21T21:26:30,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 in 170 msec 2024-11-21T21:26:30,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T21:26:30,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, REOPEN/MOVE in 489 msec 2024-11-21T21:26:30,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-21T21:26:30,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-21T21:26:30,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-21T21:26:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T21:26:30,127 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:30,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447b2b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,130 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@95aa0fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,132 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616c1dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,133 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d539e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,134 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca0cfcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f19d4e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,137 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad68e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57315279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,139 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b540e78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,140 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3bb8a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,142 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19461d47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,143 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d370b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476aa6f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,146 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4a0a17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,148 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d2af50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,150 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1523d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,152 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b3ebae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,153 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fe240f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,35047,-1 for getting cluster id 2024-11-21T21:26:30,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:26:30,154 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38eb55c4-5488-4e84-a66c-60c020ae062e' 2024-11-21T21:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38eb55c4-5488-4e84-a66c-60c020ae062e" 2024-11-21T21:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b513f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,35047,-1] 2024-11-21T21:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:30,155 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:26:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3205d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:26:30,156 DEBUG [Thread-2537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,156 DEBUG [Thread-2539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,157 DEBUG [Thread-2530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,157 DEBUG [Thread-2522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,157 DEBUG [Thread-2524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,157 DEBUG [Thread-2533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,157 DEBUG [Thread-2528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,158 DEBUG [Thread-2526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,158 DEBUG [Thread-2535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:30,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-21T21:26:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T21:26:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,163 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,164 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:30,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,165 DEBUG [Thread-2541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:26:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,34123,1732224229084, seqNum=-1] 2024-11-21T21:26:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:26:30,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:26:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5] 2024-11-21T21:26:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,202 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:30,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:26:30,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e66a6e1377c4d70bc292a6b8ab2fe3e_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224390201/Put/seqid=0 2024-11-21T21:26:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224450230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224450230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224450231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224450232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224450233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742394_1570 (size=12154) 2024-11-21T21:26:30,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:30,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e66a6e1377c4d70bc292a6b8ab2fe3e_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e66a6e1377c4d70bc292a6b8ab2fe3e_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:30,264 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/fc6ce3ed5a6a4b1da3305ba8c04adc64, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:30,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/fc6ce3ed5a6a4b1da3305ba8c04adc64 is 175, key is test_row_0/A:col10/1732224390201/Put/seqid=0 2024-11-21T21:26:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T21:26:30,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742395_1571 (size=30955) 2024-11-21T21:26:30,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T21:26:30,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:30,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224450339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224450340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224450340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224450340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224450340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T21:26:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T21:26:30,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T21:26:30,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:30,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/fc6ce3ed5a6a4b1da3305ba8c04adc64 2024-11-21T21:26:30,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/93533b9f8f4443a9b8529cb316abbc75 is 50, key is test_row_0/B:col10/1732224390201/Put/seqid=0 2024-11-21T21:26:30,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742396_1572 (size=12001) 2024-11-21T21:26:30,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T21:26:30,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T21:26:30,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224450860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224450860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224450860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224450860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224450860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:30,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T21:26:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:30,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:31,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T21:26:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:31,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:31,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/93533b9f8f4443a9b8529cb316abbc75 2024-11-21T21:26:31,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c744cb6aa2f345e9b275f0c6a556969c is 50, key is test_row_0/C:col10/1732224390201/Put/seqid=0 2024-11-21T21:26:31,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742397_1573 (size=12001) 2024-11-21T21:26:31,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c744cb6aa2f345e9b275f0c6a556969c 2024-11-21T21:26:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/fc6ce3ed5a6a4b1da3305ba8c04adc64 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fc6ce3ed5a6a4b1da3305ba8c04adc64 2024-11-21T21:26:31,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fc6ce3ed5a6a4b1da3305ba8c04adc64, entries=150, sequenceid=17, filesize=30.2 K 2024-11-21T21:26:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/93533b9f8f4443a9b8529cb316abbc75 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/93533b9f8f4443a9b8529cb316abbc75 2024-11-21T21:26:31,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/93533b9f8f4443a9b8529cb316abbc75, entries=150, sequenceid=17, filesize=11.7 K 2024-11-21T21:26:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c744cb6aa2f345e9b275f0c6a556969c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c744cb6aa2f345e9b275f0c6a556969c 2024-11-21T21:26:31,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c744cb6aa2f345e9b275f0c6a556969c, entries=150, sequenceid=17, filesize=11.7 K 2024-11-21T21:26:31,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2ddd2a6cb469a423cd30254b88184ba0 in 951ms, sequenceid=17, compaction requested=false 2024-11-21T21:26:31,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:31,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T21:26:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,233 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:26:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215287ac8d637b461d8b08095484ff697e_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224390230/Put/seqid=0 2024-11-21T21:26:31,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742398_1574 (size=12154) 2024-11-21T21:26:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,258 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215287ac8d637b461d8b08095484ff697e_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215287ac8d637b461d8b08095484ff697e_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f49b4ccf39424b79a186b4fb650fe078, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f49b4ccf39424b79a186b4fb650fe078 is 175, key is test_row_0/A:col10/1732224390230/Put/seqid=0 2024-11-21T21:26:31,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742399_1575 (size=30955) 2024-11-21T21:26:31,278 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f49b4ccf39424b79a186b4fb650fe078 2024-11-21T21:26:31,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/e6bad08e07ec4daba9453d5cf9dd2f3e is 50, key is test_row_0/B:col10/1732224390230/Put/seqid=0 2024-11-21T21:26:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T21:26:31,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742400_1576 (size=12001) 2024-11-21T21:26:31,304 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/e6bad08e07ec4daba9453d5cf9dd2f3e 2024-11-21T21:26:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/d070fb1a39da4fb2865f28795fca250f is 50, key is test_row_0/C:col10/1732224390230/Put/seqid=0 2024-11-21T21:26:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742401_1577 (size=12001) 2024-11-21T21:26:31,322 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/d070fb1a39da4fb2865f28795fca250f 2024-11-21T21:26:31,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f49b4ccf39424b79a186b4fb650fe078 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f49b4ccf39424b79a186b4fb650fe078 2024-11-21T21:26:31,331 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f49b4ccf39424b79a186b4fb650fe078, entries=150, sequenceid=40, filesize=30.2 K 2024-11-21T21:26:31,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/e6bad08e07ec4daba9453d5cf9dd2f3e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/e6bad08e07ec4daba9453d5cf9dd2f3e 2024-11-21T21:26:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,340 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/e6bad08e07ec4daba9453d5cf9dd2f3e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/d070fb1a39da4fb2865f28795fca250f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/d070fb1a39da4fb2865f28795fca250f 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,344 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/d070fb1a39da4fb2865f28795fca250f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T21:26:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,345 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 2ddd2a6cb469a423cd30254b88184ba0 in 112ms, sequenceid=40, compaction requested=false 2024-11-21T21:26:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-21T21:26:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-21T21:26:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T21:26:31,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1820 sec 2024-11-21T21:26:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.1880 sec 2024-11-21T21:26:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T21:26:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f316fb9760fa4694b61868f6c09c0cd3_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224391398/Put/seqid=0 2024-11-21T21:26:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742403_1579 (size=24358) 2024-11-21T21:26:31,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f316fb9760fa4694b61868f6c09c0cd3_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f316fb9760fa4694b61868f6c09c0cd3_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:31,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/33f54441c1f341308830541906f03c78, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:31,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/33f54441c1f341308830541906f03c78 is 175, key is test_row_0/A:col10/1732224391398/Put/seqid=0 2024-11-21T21:26:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742402_1578 (size=73995) 2024-11-21T21:26:31,431 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/33f54441c1f341308830541906f03c78 2024-11-21T21:26:31,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/913cb16ebd5d49d18a532518b0ae1d76 is 50, key is test_row_0/B:col10/1732224391398/Put/seqid=0 2024-11-21T21:26:31,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742404_1580 (size=12001) 2024-11-21T21:26:31,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/913cb16ebd5d49d18a532518b0ae1d76 2024-11-21T21:26:31,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224451446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224451447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224451457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/48f72f30554149c9977f5f1b641790f6 is 50, key is test_row_0/C:col10/1732224391398/Put/seqid=0 2024-11-21T21:26:31,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224451458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224451458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742405_1581 (size=12001) 2024-11-21T21:26:31,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/48f72f30554149c9977f5f1b641790f6 2024-11-21T21:26:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/33f54441c1f341308830541906f03c78 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/33f54441c1f341308830541906f03c78 2024-11-21T21:26:31,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/33f54441c1f341308830541906f03c78, entries=400, sequenceid=51, filesize=72.3 K 2024-11-21T21:26:31,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/913cb16ebd5d49d18a532518b0ae1d76 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/913cb16ebd5d49d18a532518b0ae1d76 2024-11-21T21:26:31,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/913cb16ebd5d49d18a532518b0ae1d76, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T21:26:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/48f72f30554149c9977f5f1b641790f6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/48f72f30554149c9977f5f1b641790f6 2024-11-21T21:26:31,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/48f72f30554149c9977f5f1b641790f6, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T21:26:31,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 2ddd2a6cb469a423cd30254b88184ba0 in 87ms, sequenceid=51, compaction requested=true 2024-11-21T21:26:31,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:31,496 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:31,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:31,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:31,496 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:31,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:31,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:31,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:31,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:31,497 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:31,497 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:31,497 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,497 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fc6ce3ed5a6a4b1da3305ba8c04adc64, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f49b4ccf39424b79a186b4fb650fe078, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/33f54441c1f341308830541906f03c78] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=132.7 K 2024-11-21T21:26:31,497 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,498 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fc6ce3ed5a6a4b1da3305ba8c04adc64, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f49b4ccf39424b79a186b4fb650fe078, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/33f54441c1f341308830541906f03c78] 2024-11-21T21:26:31,499 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6ce3ed5a6a4b1da3305ba8c04adc64, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732224390201 2024-11-21T21:26:31,499 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:31,499 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f49b4ccf39424b79a186b4fb650fe078, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732224390224 2024-11-21T21:26:31,499 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:31,499 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,499 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/93533b9f8f4443a9b8529cb316abbc75, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/e6bad08e07ec4daba9453d5cf9dd2f3e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/913cb16ebd5d49d18a532518b0ae1d76] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.2 K 2024-11-21T21:26:31,499 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33f54441c1f341308830541906f03c78, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224391395 2024-11-21T21:26:31,500 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 93533b9f8f4443a9b8529cb316abbc75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732224390201 2024-11-21T21:26:31,500 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting e6bad08e07ec4daba9453d5cf9dd2f3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732224390224 2024-11-21T21:26:31,500 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 913cb16ebd5d49d18a532518b0ae1d76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224391398 2024-11-21T21:26:31,513 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:31,513 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/aec2edaf5c9c400b9f3c9623ffe571f4 is 50, key is test_row_0/B:col10/1732224391398/Put/seqid=0 2024-11-21T21:26:31,517 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:31,522 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211bfa504b323b417dafe398aee4405059_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:31,524 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211bfa504b323b417dafe398aee4405059_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:31,524 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211bfa504b323b417dafe398aee4405059_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:31,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742407_1583 (size=4469) 2024-11-21T21:26:31,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742406_1582 (size=12104) 2024-11-21T21:26:31,536 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#496 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:31,537 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/4f337d5742864dffba0639086ec60a27 is 175, key is test_row_0/A:col10/1732224391398/Put/seqid=0 2024-11-21T21:26:31,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742408_1584 (size=31058) 2024-11-21T21:26:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:31,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:31,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T21:26:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224451574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224451574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224451575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224451577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112172a130b1c0f242a4ad2d5064578c7d6a_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224391572/Put/seqid=0 2024-11-21T21:26:31,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224451578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742409_1585 (size=14594) 2024-11-21T21:26:31,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:31,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112172a130b1c0f242a4ad2d5064578c7d6a_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112172a130b1c0f242a4ad2d5064578c7d6a_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:31,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/e235dbd786b9456b8b30a327574e96e5, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:31,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/e235dbd786b9456b8b30a327574e96e5 is 175, key is test_row_0/A:col10/1732224391572/Put/seqid=0 2024-11-21T21:26:31,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742410_1586 (size=39549) 2024-11-21T21:26:31,605 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/e235dbd786b9456b8b30a327574e96e5 2024-11-21T21:26:31,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9fd987c04a6b47c380205be321e7450d is 50, key is test_row_0/B:col10/1732224391572/Put/seqid=0 2024-11-21T21:26:31,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742411_1587 (size=12001) 2024-11-21T21:26:31,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224451679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224451690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224451690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224451690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224451690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,775 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:26:31,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224451889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224451900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224451900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224451900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224451900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:31,940 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/aec2edaf5c9c400b9f3c9623ffe571f4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/aec2edaf5c9c400b9f3c9623ffe571f4 2024-11-21T21:26:31,945 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into aec2edaf5c9c400b9f3c9623ffe571f4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:31,945 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:31,945 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=13, startTime=1732224391496; duration=0sec 2024-11-21T21:26:31,945 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:31,945 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:31,945 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:31,946 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:31,946 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:31,946 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:31,946 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c744cb6aa2f345e9b275f0c6a556969c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/d070fb1a39da4fb2865f28795fca250f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/48f72f30554149c9977f5f1b641790f6] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.2 K 2024-11-21T21:26:31,947 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c744cb6aa2f345e9b275f0c6a556969c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732224390201 2024-11-21T21:26:31,947 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting d070fb1a39da4fb2865f28795fca250f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732224390224 2024-11-21T21:26:31,947 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f72f30554149c9977f5f1b641790f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224391398 2024-11-21T21:26:31,960 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/4f337d5742864dffba0639086ec60a27 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/4f337d5742864dffba0639086ec60a27 2024-11-21T21:26:31,965 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into 4f337d5742864dffba0639086ec60a27(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:31,966 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:31,966 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=13, startTime=1732224391496; duration=0sec 2024-11-21T21:26:31,966 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:31,966 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:31,970 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:31,971 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/34cd0395aabe4c3697b9355749ee45c2 is 50, key is test_row_0/C:col10/1732224391398/Put/seqid=0 2024-11-21T21:26:31,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742412_1588 (size=12104) 2024-11-21T21:26:32,001 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/34cd0395aabe4c3697b9355749ee45c2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/34cd0395aabe4c3697b9355749ee45c2 2024-11-21T21:26:32,005 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into 34cd0395aabe4c3697b9355749ee45c2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:32,005 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:32,005 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=13, startTime=1732224391497; duration=0sec 2024-11-21T21:26:32,005 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:32,005 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:32,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9fd987c04a6b47c380205be321e7450d 2024-11-21T21:26:32,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/b7c63cb82cfd45d1ad253da3fd2c80c8 is 50, key is test_row_0/C:col10/1732224391572/Put/seqid=0 2024-11-21T21:26:32,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742413_1589 (size=12001) 2024-11-21T21:26:32,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/b7c63cb82cfd45d1ad253da3fd2c80c8 2024-11-21T21:26:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/e235dbd786b9456b8b30a327574e96e5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/e235dbd786b9456b8b30a327574e96e5 2024-11-21T21:26:32,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/e235dbd786b9456b8b30a327574e96e5, entries=200, sequenceid=79, filesize=38.6 K 2024-11-21T21:26:32,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9fd987c04a6b47c380205be321e7450d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9fd987c04a6b47c380205be321e7450d 2024-11-21T21:26:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9fd987c04a6b47c380205be321e7450d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T21:26:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/b7c63cb82cfd45d1ad253da3fd2c80c8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b7c63cb82cfd45d1ad253da3fd2c80c8 2024-11-21T21:26:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b7c63cb82cfd45d1ad253da3fd2c80c8, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T21:26:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 2ddd2a6cb469a423cd30254b88184ba0 in 480ms, sequenceid=79, compaction requested=false 2024-11-21T21:26:32,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:32,213 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:32,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T21:26:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112187836e5ce2b64c4d88892e654366af03_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224392212/Put/seqid=0 2024-11-21T21:26:32,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742414_1590 (size=21918) 2024-11-21T21:26:32,230 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:32,240 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112187836e5ce2b64c4d88892e654366af03_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112187836e5ce2b64c4d88892e654366af03_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:32,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/21a1cf4d7b524b359b2865a75225be2f, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:32,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/21a1cf4d7b524b359b2865a75225be2f is 175, key is test_row_0/A:col10/1732224392212/Put/seqid=0 2024-11-21T21:26:32,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742415_1591 (size=65323) 2024-11-21T21:26:32,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224452270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224452270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224452271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224452274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224452277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T21:26:32,310 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:32,310 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-21T21:26:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T21:26:32,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:32,312 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:32,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224452380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224452380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224452380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224452389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224452390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T21:26:32,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:32,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:32,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224452589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224452589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224452589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224452599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224452600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:32,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:32,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T21:26:32,648 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/21a1cf4d7b524b359b2865a75225be2f 2024-11-21T21:26:32,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/3fa67289c79a42369691db640b9172e7 is 50, key is test_row_0/B:col10/1732224392212/Put/seqid=0 2024-11-21T21:26:32,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742416_1592 (size=12001) 2024-11-21T21:26:32,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:32,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224452899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224452899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224452899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224452909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224452910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:32,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:32,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T21:26:33,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/3fa67289c79a42369691db640b9172e7 2024-11-21T21:26:33,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c2ec01eaa8424ea5b737b6a4bdb6f835 is 50, key is test_row_0/C:col10/1732224392212/Put/seqid=0 2024-11-21T21:26:33,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:33,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:33,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742417_1593 (size=12001) 2024-11-21T21:26:33,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:33,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:33,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:33,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:33,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:33,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224453409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224453410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224453410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:33,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224453420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:33,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224453420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T21:26:33,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c2ec01eaa8424ea5b737b6a4bdb6f835 2024-11-21T21:26:33,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/21a1cf4d7b524b359b2865a75225be2f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/21a1cf4d7b524b359b2865a75225be2f 2024-11-21T21:26:33,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/21a1cf4d7b524b359b2865a75225be2f, entries=350, sequenceid=95, filesize=63.8 K 2024-11-21T21:26:33,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/3fa67289c79a42369691db640b9172e7 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3fa67289c79a42369691db640b9172e7 2024-11-21T21:26:33,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3fa67289c79a42369691db640b9172e7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T21:26:33,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c2ec01eaa8424ea5b737b6a4bdb6f835 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c2ec01eaa8424ea5b737b6a4bdb6f835 2024-11-21T21:26:33,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c2ec01eaa8424ea5b737b6a4bdb6f835, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T21:26:33,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2ddd2a6cb469a423cd30254b88184ba0 in 1280ms, sequenceid=95, compaction requested=true 2024-11-21T21:26:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:33,493 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:33,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:33,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:33,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:33,494 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:33,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:33,494 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135930 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:33,494 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:33,494 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,495 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/4f337d5742864dffba0639086ec60a27, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/e235dbd786b9456b8b30a327574e96e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/21a1cf4d7b524b359b2865a75225be2f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=132.7 K 2024-11-21T21:26:33,495 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,495 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/4f337d5742864dffba0639086ec60a27, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/e235dbd786b9456b8b30a327574e96e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/21a1cf4d7b524b359b2865a75225be2f] 2024-11-21T21:26:33,495 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:33,495 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:33,495 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,495 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/aec2edaf5c9c400b9f3c9623ffe571f4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9fd987c04a6b47c380205be321e7450d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3fa67289c79a42369691db640b9172e7] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.3 K 2024-11-21T21:26:33,495 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f337d5742864dffba0639086ec60a27, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224391398 2024-11-21T21:26:33,496 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting aec2edaf5c9c400b9f3c9623ffe571f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224391398 2024-11-21T21:26:33,496 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e235dbd786b9456b8b30a327574e96e5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732224391444 2024-11-21T21:26:33,496 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a1cf4d7b524b359b2865a75225be2f, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732224391575 2024-11-21T21:26:33,496 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fd987c04a6b47c380205be321e7450d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732224391444 2024-11-21T21:26:33,497 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa67289c79a42369691db640b9172e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732224392209 2024-11-21T21:26:33,505 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:33,522 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:33,523 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/172bddae30df47ba8bfea1359ae5ab3f is 50, key is test_row_0/B:col10/1732224392212/Put/seqid=0 2024-11-21T21:26:33,524 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411215adc645993d94a8b85af9dc37aaa40f5_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:33,526 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411215adc645993d94a8b85af9dc37aaa40f5_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:33,526 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215adc645993d94a8b85af9dc37aaa40f5_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:33,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T21:26:33,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,533 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T21:26:33,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742418_1594 (size=12207) 2024-11-21T21:26:33,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742419_1595 (size=4469) 2024-11-21T21:26:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214a859354adac46b68f2ba001fb500761_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224392275/Put/seqid=0 2024-11-21T21:26:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742420_1596 (size=12154) 2024-11-21T21:26:33,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:33,592 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214a859354adac46b68f2ba001fb500761_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214a859354adac46b68f2ba001fb500761_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/cf6f971dc5ac4c288b1a88d1c37d08c3, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/cf6f971dc5ac4c288b1a88d1c37d08c3 is 175, key is test_row_0/A:col10/1732224392275/Put/seqid=0 2024-11-21T21:26:33,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742421_1597 (size=30955) 2024-11-21T21:26:33,944 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/172bddae30df47ba8bfea1359ae5ab3f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/172bddae30df47ba8bfea1359ae5ab3f 2024-11-21T21:26:33,947 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into 172bddae30df47ba8bfea1359ae5ab3f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:33,948 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:33,948 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=13, startTime=1732224393493; duration=0sec 2024-11-21T21:26:33,948 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:33,948 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:33,948 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:33,948 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:33,948 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:33,948 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:33,949 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/34cd0395aabe4c3697b9355749ee45c2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b7c63cb82cfd45d1ad253da3fd2c80c8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c2ec01eaa8424ea5b737b6a4bdb6f835] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.3 K 2024-11-21T21:26:33,949 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 34cd0395aabe4c3697b9355749ee45c2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732224391398 2024-11-21T21:26:33,949 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b7c63cb82cfd45d1ad253da3fd2c80c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732224391444 2024-11-21T21:26:33,949 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ec01eaa8424ea5b737b6a4bdb6f835, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732224392209 2024-11-21T21:26:33,954 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:33,954 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/742eadbb44fe4da2b74082849b83917a is 50, key is test_row_0/C:col10/1732224392212/Put/seqid=0 2024-11-21T21:26:33,963 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#504 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:33,963 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/bb8a37ad16fd482cad1f92dd43d9bedd is 175, key is test_row_0/A:col10/1732224392212/Put/seqid=0 2024-11-21T21:26:33,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742423_1599 (size=31161) 2024-11-21T21:26:33,976 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/bb8a37ad16fd482cad1f92dd43d9bedd as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/bb8a37ad16fd482cad1f92dd43d9bedd 2024-11-21T21:26:33,982 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into bb8a37ad16fd482cad1f92dd43d9bedd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:33,983 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:33,983 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=13, startTime=1732224393493; duration=0sec 2024-11-21T21:26:33,983 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:33,983 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:34,011 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/cf6f971dc5ac4c288b1a88d1c37d08c3 2024-11-21T21:26:34,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/70caf8a326944a9db960797ddd37b4d5 is 50, key is test_row_0/B:col10/1732224392275/Put/seqid=0 2024-11-21T21:26:34,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742422_1598 (size=12207) 2024-11-21T21:26:34,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742424_1600 (size=12001) 2024-11-21T21:26:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:34,439 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/70caf8a326944a9db960797ddd37b4d5 2024-11-21T21:26:34,439 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/742eadbb44fe4da2b74082849b83917a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/742eadbb44fe4da2b74082849b83917a 2024-11-21T21:26:34,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224454434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224454435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224454438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224454440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,442 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,444 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into 742eadbb44fe4da2b74082849b83917a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:34,444 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:34,444 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=13, startTime=1732224393494; duration=0sec 2024-11-21T21:26:34,444 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:34,444 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:34,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224454440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,445 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/7977b009a37248b2a573f9e46ebc27dc is 50, key is test_row_0/C:col10/1732224392275/Put/seqid=0 2024-11-21T21:26:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T21:26:34,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742425_1601 (size=12001) 2024-11-21T21:26:34,461 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/7977b009a37248b2a573f9e46ebc27dc 2024-11-21T21:26:34,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/cf6f971dc5ac4c288b1a88d1c37d08c3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/cf6f971dc5ac4c288b1a88d1c37d08c3 2024-11-21T21:26:34,470 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/cf6f971dc5ac4c288b1a88d1c37d08c3, entries=150, sequenceid=118, filesize=30.2 K 2024-11-21T21:26:34,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/70caf8a326944a9db960797ddd37b4d5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/70caf8a326944a9db960797ddd37b4d5 2024-11-21T21:26:34,475 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/70caf8a326944a9db960797ddd37b4d5, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T21:26:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/7977b009a37248b2a573f9e46ebc27dc as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7977b009a37248b2a573f9e46ebc27dc 2024-11-21T21:26:34,480 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7977b009a37248b2a573f9e46ebc27dc, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T21:26:34,480 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 2ddd2a6cb469a423cd30254b88184ba0 in 947ms, sequenceid=118, compaction requested=false 2024-11-21T21:26:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:34,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-21T21:26:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-21T21:26:34,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-21T21:26:34,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-21T21:26:34,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.1750 sec 2024-11-21T21:26:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:34,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T21:26:34,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121286f58801d29479a8f9fd968bd97a79a_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224394550/Put/seqid=0 2024-11-21T21:26:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742426_1602 (size=17284) 2024-11-21T21:26:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224454577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224454578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224454579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224454689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224454689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224454689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224454899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224454899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224454899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:34,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:34,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121286f58801d29479a8f9fd968bd97a79a_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121286f58801d29479a8f9fd968bd97a79a_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:34,964 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/b59f246fc0f34a3793b95d8944218ba4, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:34,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/b59f246fc0f34a3793b95d8944218ba4 is 175, key is test_row_0/A:col10/1732224394550/Put/seqid=0 2024-11-21T21:26:34,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742427_1603 (size=48389) 2024-11-21T21:26:34,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/b59f246fc0f34a3793b95d8944218ba4 2024-11-21T21:26:34,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/fdaa0b184bda455aa133224653f6762f is 50, key is test_row_0/B:col10/1732224394550/Put/seqid=0 2024-11-21T21:26:35,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742428_1604 (size=12151) 2024-11-21T21:26:35,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/fdaa0b184bda455aa133224653f6762f 2024-11-21T21:26:35,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4318f102fb0d47008dfc3b3c37bef3a0 is 50, key is test_row_0/C:col10/1732224394550/Put/seqid=0 2024-11-21T21:26:35,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742429_1605 (size=12151) 2024-11-21T21:26:35,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4318f102fb0d47008dfc3b3c37bef3a0 2024-11-21T21:26:35,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/b59f246fc0f34a3793b95d8944218ba4 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b59f246fc0f34a3793b95d8944218ba4 2024-11-21T21:26:35,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b59f246fc0f34a3793b95d8944218ba4, entries=250, sequenceid=136, filesize=47.3 K 2024-11-21T21:26:35,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/fdaa0b184bda455aa133224653f6762f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/fdaa0b184bda455aa133224653f6762f 2024-11-21T21:26:35,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/fdaa0b184bda455aa133224653f6762f, entries=150, sequenceid=136, filesize=11.9 K 2024-11-21T21:26:35,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4318f102fb0d47008dfc3b3c37bef3a0 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4318f102fb0d47008dfc3b3c37bef3a0 2024-11-21T21:26:35,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4318f102fb0d47008dfc3b3c37bef3a0, entries=150, sequenceid=136, filesize=11.9 K 2024-11-21T21:26:35,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2ddd2a6cb469a423cd30254b88184ba0 in 519ms, sequenceid=136, compaction requested=true 2024-11-21T21:26:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:35,070 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:35,070 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:35,071 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:35,071 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:35,071 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:35,071 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/bb8a37ad16fd482cad1f92dd43d9bedd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/cf6f971dc5ac4c288b1a88d1c37d08c3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b59f246fc0f34a3793b95d8944218ba4] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=107.9 K 2024-11-21T21:26:35,071 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:35,071 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/bb8a37ad16fd482cad1f92dd43d9bedd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/cf6f971dc5ac4c288b1a88d1c37d08c3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b59f246fc0f34a3793b95d8944218ba4] 2024-11-21T21:26:35,072 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb8a37ad16fd482cad1f92dd43d9bedd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732224392209 2024-11-21T21:26:35,072 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:35,072 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:35,072 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:35,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:35,072 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/172bddae30df47ba8bfea1359ae5ab3f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/70caf8a326944a9db960797ddd37b4d5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/fdaa0b184bda455aa133224653f6762f] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.5 K 2024-11-21T21:26:35,073 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf6f971dc5ac4c288b1a88d1c37d08c3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732224392241 2024-11-21T21:26:35,073 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 172bddae30df47ba8bfea1359ae5ab3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732224392209 2024-11-21T21:26:35,073 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59f246fc0f34a3793b95d8944218ba4, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732224394433 2024-11-21T21:26:35,073 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 70caf8a326944a9db960797ddd37b4d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732224392241 2024-11-21T21:26:35,074 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting fdaa0b184bda455aa133224653f6762f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732224394433 2024-11-21T21:26:35,081 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:35,086 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:35,087 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/06e9b9a179704f39bec85fad0149dc2d is 50, key is test_row_0/B:col10/1732224394550/Put/seqid=0 2024-11-21T21:26:35,101 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ae6412b4fbe543a1bb588b4312916d27_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:35,102 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ae6412b4fbe543a1bb588b4312916d27_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:35,103 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ae6412b4fbe543a1bb588b4312916d27_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:35,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742430_1606 (size=12459) 2024-11-21T21:26:35,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742431_1607 (size=4469) 2024-11-21T21:26:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:35,211 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:35,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T21:26:35,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215f376574e7f0436499e333aabee75d5b_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224395210/Put/seqid=0 2024-11-21T21:26:35,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742432_1608 (size=12304) 2024-11-21T21:26:35,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224455225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224455228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224455231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224455339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224455339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224455339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,539 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#513 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:35,539 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f08413bb32764831aba68b4d67e6d037 is 175, key is test_row_0/A:col10/1732224394550/Put/seqid=0 2024-11-21T21:26:35,542 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/06e9b9a179704f39bec85fad0149dc2d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/06e9b9a179704f39bec85fad0149dc2d 2024-11-21T21:26:35,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742433_1609 (size=31413) 2024-11-21T21:26:35,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224455549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224455549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224455549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,555 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into 06e9b9a179704f39bec85fad0149dc2d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:35,555 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:35,555 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=13, startTime=1732224395070; duration=0sec 2024-11-21T21:26:35,555 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:35,555 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:35,555 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:35,556 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f08413bb32764831aba68b4d67e6d037 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f08413bb32764831aba68b4d67e6d037 2024-11-21T21:26:35,559 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:35,560 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:35,560 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:35,560 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/742eadbb44fe4da2b74082849b83917a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7977b009a37248b2a573f9e46ebc27dc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4318f102fb0d47008dfc3b3c37bef3a0] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.5 K 2024-11-21T21:26:35,560 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 742eadbb44fe4da2b74082849b83917a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732224392209 2024-11-21T21:26:35,561 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7977b009a37248b2a573f9e46ebc27dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732224392241 2024-11-21T21:26:35,561 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4318f102fb0d47008dfc3b3c37bef3a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732224394433 2024-11-21T21:26:35,562 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into f08413bb32764831aba68b4d67e6d037(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:35,562 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:35,562 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=13, startTime=1732224395070; duration=0sec 2024-11-21T21:26:35,562 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:35,562 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:35,574 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:35,574 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/3ff92a6d99144e66851f8959b90914ba is 50, key is test_row_0/C:col10/1732224394550/Put/seqid=0 2024-11-21T21:26:35,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742434_1610 (size=12459) 2024-11-21T21:26:35,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:35,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215f376574e7f0436499e333aabee75d5b_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215f376574e7f0436499e333aabee75d5b_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:35,633 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f62de3340a3a4ddc8d350daac7a79545, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:35,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f62de3340a3a4ddc8d350daac7a79545 is 175, key is test_row_0/A:col10/1732224395210/Put/seqid=0 2024-11-21T21:26:35,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742435_1611 (size=31105) 2024-11-21T21:26:35,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f62de3340a3a4ddc8d350daac7a79545 2024-11-21T21:26:35,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/5ff97aa071b24cada4e2781275728ac1 is 50, key is test_row_0/B:col10/1732224395210/Put/seqid=0 2024-11-21T21:26:35,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742436_1612 (size=12151) 2024-11-21T21:26:35,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224455860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224455860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224455860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:35,986 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/3ff92a6d99144e66851f8959b90914ba as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/3ff92a6d99144e66851f8959b90914ba 2024-11-21T21:26:35,999 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into 3ff92a6d99144e66851f8959b90914ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:35,999 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:35,999 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=13, startTime=1732224395070; duration=0sec 2024-11-21T21:26:35,999 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:35,999 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:36,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/5ff97aa071b24cada4e2781275728ac1 2024-11-21T21:26:36,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c29e4619d1f449829e5358256ce151ba is 50, key is test_row_0/C:col10/1732224395210/Put/seqid=0 2024-11-21T21:26:36,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742437_1613 (size=12151) 2024-11-21T21:26:36,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c29e4619d1f449829e5358256ce151ba 2024-11-21T21:26:36,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/f62de3340a3a4ddc8d350daac7a79545 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f62de3340a3a4ddc8d350daac7a79545 2024-11-21T21:26:36,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f62de3340a3a4ddc8d350daac7a79545, entries=150, sequenceid=158, filesize=30.4 K 2024-11-21T21:26:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/5ff97aa071b24cada4e2781275728ac1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/5ff97aa071b24cada4e2781275728ac1 2024-11-21T21:26:36,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/5ff97aa071b24cada4e2781275728ac1, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T21:26:36,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/c29e4619d1f449829e5358256ce151ba as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c29e4619d1f449829e5358256ce151ba 2024-11-21T21:26:36,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c29e4619d1f449829e5358256ce151ba, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T21:26:36,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2ddd2a6cb469a423cd30254b88184ba0 in 868ms, sequenceid=158, compaction requested=false 2024-11-21T21:26:36,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:36,372 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:36,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T21:26:36,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d02114ee3d7c46bcaba75a8e893fb08c_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224396371/Put/seqid=0 2024-11-21T21:26:36,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742438_1614 (size=14794) 2024-11-21T21:26:36,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:36,384 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d02114ee3d7c46bcaba75a8e893fb08c_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d02114ee3d7c46bcaba75a8e893fb08c_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:36,385 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/3c0d1de2e15a4d2fa9624762c447cc55, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:36,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/3c0d1de2e15a4d2fa9624762c447cc55 is 175, key is test_row_0/A:col10/1732224396371/Put/seqid=0 2024-11-21T21:26:36,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742439_1615 (size=39749) 2024-11-21T21:26:36,388 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/3c0d1de2e15a4d2fa9624762c447cc55 2024-11-21T21:26:36,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9b2403a9c93c4e489f3842375b19fe2c is 50, key is test_row_0/B:col10/1732224396371/Put/seqid=0 2024-11-21T21:26:36,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742440_1616 (size=12151) 2024-11-21T21:26:36,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224456401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224456403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224456403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224456460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224456460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,463 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,463 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T21:26:36,469 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:36,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-21T21:26:36,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:26:36,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:36,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:36,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224456510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224456510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224456510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:26:36,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T21:26:36,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:36,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:36,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:36,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224456719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224456720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224456720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:36,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T21:26:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:36,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:26:36,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9b2403a9c93c4e489f3842375b19fe2c 2024-11-21T21:26:36,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/b5ad2e6b129e446e9f0beb610a505173 is 50, key is test_row_0/C:col10/1732224396371/Put/seqid=0 2024-11-21T21:26:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742441_1617 (size=12151) 2024-11-21T21:26:36,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T21:26:36,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:36,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:36,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:37,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224457029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224457029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224457029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T21:26:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:26:37,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/b5ad2e6b129e446e9f0beb610a505173 2024-11-21T21:26:37,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/3c0d1de2e15a4d2fa9624762c447cc55 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/3c0d1de2e15a4d2fa9624762c447cc55 2024-11-21T21:26:37,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/3c0d1de2e15a4d2fa9624762c447cc55, entries=200, sequenceid=177, filesize=38.8 K 2024-11-21T21:26:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9b2403a9c93c4e489f3842375b19fe2c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9b2403a9c93c4e489f3842375b19fe2c 2024-11-21T21:26:37,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9b2403a9c93c4e489f3842375b19fe2c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-21T21:26:37,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/b5ad2e6b129e446e9f0beb610a505173 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b5ad2e6b129e446e9f0beb610a505173 2024-11-21T21:26:37,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T21:26:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:37,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:37,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b5ad2e6b129e446e9f0beb610a505173, entries=150, sequenceid=177, filesize=11.9 K 2024-11-21T21:26:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:37,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2ddd2a6cb469a423cd30254b88184ba0 in 865ms, sequenceid=177, compaction requested=true 2024-11-21T21:26:37,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:37,237 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:37,238 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:37,241 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:37,241 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:37,241 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:37,241 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:37,241 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,241 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,241 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f08413bb32764831aba68b4d67e6d037, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f62de3340a3a4ddc8d350daac7a79545, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/3c0d1de2e15a4d2fa9624762c447cc55] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=99.9 K 2024-11-21T21:26:37,241 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/06e9b9a179704f39bec85fad0149dc2d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/5ff97aa071b24cada4e2781275728ac1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9b2403a9c93c4e489f3842375b19fe2c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.9 K 2024-11-21T21:26:37,241 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,241 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f08413bb32764831aba68b4d67e6d037, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f62de3340a3a4ddc8d350daac7a79545, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/3c0d1de2e15a4d2fa9624762c447cc55] 2024-11-21T21:26:37,242 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f08413bb32764831aba68b4d67e6d037, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732224394433 2024-11-21T21:26:37,242 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e9b9a179704f39bec85fad0149dc2d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732224394433 2024-11-21T21:26:37,242 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f62de3340a3a4ddc8d350daac7a79545, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732224394573 2024-11-21T21:26:37,242 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ff97aa071b24cada4e2781275728ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732224394573 2024-11-21T21:26:37,242 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b2403a9c93c4e489f3842375b19fe2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732224395229 2024-11-21T21:26:37,242 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c0d1de2e15a4d2fa9624762c447cc55, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732224395224 2024-11-21T21:26:37,246 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:37,247 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:37,248 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/041d75aef9e844acb359a412f9e9f54c is 50, key is test_row_0/B:col10/1732224396371/Put/seqid=0 2024-11-21T21:26:37,248 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112158431f7c853e4ae997ce942d612d351e_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:37,250 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112158431f7c853e4ae997ce942d612d351e_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:37,250 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112158431f7c853e4ae997ce942d612d351e_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:37,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742442_1618 (size=12561) 2024-11-21T21:26:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742443_1619 (size=4469) 2024-11-21T21:26:37,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T21:26:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,388 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T21:26:37,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112111252c972f71458a8f9c964d2af7c2b5_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224396398/Put/seqid=0 2024-11-21T21:26:37,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742444_1620 (size=12304) 2024-11-21T21:26:37,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:37,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224457559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224457561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224457562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:26:37,669 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/041d75aef9e844acb359a412f9e9f54c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/041d75aef9e844acb359a412f9e9f54c 2024-11-21T21:26:37,673 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into 041d75aef9e844acb359a412f9e9f54c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:37,673 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:37,673 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=13, startTime=1732224397238; duration=0sec 2024-11-21T21:26:37,673 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:37,673 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:37,673 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224457670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224457670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224457670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,676 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:37,676 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:37,677 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:37,677 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/3ff92a6d99144e66851f8959b90914ba, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c29e4619d1f449829e5358256ce151ba, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b5ad2e6b129e446e9f0beb610a505173] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=35.9 K 2024-11-21T21:26:37,677 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ff92a6d99144e66851f8959b90914ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732224394433 2024-11-21T21:26:37,677 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting c29e4619d1f449829e5358256ce151ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732224394573 2024-11-21T21:26:37,677 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ad2e6b129e446e9f0beb610a505173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732224395229 2024-11-21T21:26:37,681 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:37,682 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/ca118fe4dd7a4ab0bc3c3e98440b6db3 is 50, key is test_row_0/C:col10/1732224396371/Put/seqid=0 2024-11-21T21:26:37,686 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#523 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:37,687 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/19ea40b5b1d5480bb3fa11c185b18244 is 175, key is test_row_0/A:col10/1732224396371/Put/seqid=0 2024-11-21T21:26:37,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742445_1621 (size=12561) 2024-11-21T21:26:37,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742446_1622 (size=31515) 2024-11-21T21:26:37,701 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/ca118fe4dd7a4ab0bc3c3e98440b6db3 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ca118fe4dd7a4ab0bc3c3e98440b6db3 2024-11-21T21:26:37,704 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into ca118fe4dd7a4ab0bc3c3e98440b6db3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:37,704 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:37,704 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=13, startTime=1732224397238; duration=0sec 2024-11-21T21:26:37,705 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:37,705 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:37,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:37,807 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112111252c972f71458a8f9c964d2af7c2b5_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112111252c972f71458a8f9c964d2af7c2b5_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/0248bea07a814f22b33402f1be93a08e, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/0248bea07a814f22b33402f1be93a08e is 175, key is test_row_0/A:col10/1732224396398/Put/seqid=0 2024-11-21T21:26:37,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742447_1623 (size=31105) 2024-11-21T21:26:37,811 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/0248bea07a814f22b33402f1be93a08e 2024-11-21T21:26:37,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/23fffe7ce89a42c585b757a303356036 is 50, key is test_row_0/B:col10/1732224396398/Put/seqid=0 2024-11-21T21:26:37,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742448_1624 (size=12151) 2024-11-21T21:26:37,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224457879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224457880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224457880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,101 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/19ea40b5b1d5480bb3fa11c185b18244 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/19ea40b5b1d5480bb3fa11c185b18244 2024-11-21T21:26:38,104 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into 19ea40b5b1d5480bb3fa11c185b18244(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:38,104 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:38,104 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=13, startTime=1732224397237; duration=0sec 2024-11-21T21:26:38,104 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:38,104 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224458189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224458189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224458189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,239 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/23fffe7ce89a42c585b757a303356036 2024-11-21T21:26:38,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/7625cb26c9264eb5a84fd5ecf29805ae is 50, key is test_row_0/C:col10/1732224396398/Put/seqid=0 2024-11-21T21:26:38,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742449_1625 (size=12151) 2024-11-21T21:26:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:26:38,663 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/7625cb26c9264eb5a84fd5ecf29805ae 2024-11-21T21:26:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/0248bea07a814f22b33402f1be93a08e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/0248bea07a814f22b33402f1be93a08e 2024-11-21T21:26:38,669 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/0248bea07a814f22b33402f1be93a08e, entries=150, sequenceid=197, filesize=30.4 K 2024-11-21T21:26:38,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/23fffe7ce89a42c585b757a303356036 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/23fffe7ce89a42c585b757a303356036 2024-11-21T21:26:38,671 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/23fffe7ce89a42c585b757a303356036, entries=150, sequenceid=197, filesize=11.9 K 2024-11-21T21:26:38,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/7625cb26c9264eb5a84fd5ecf29805ae as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7625cb26c9264eb5a84fd5ecf29805ae 2024-11-21T21:26:38,674 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7625cb26c9264eb5a84fd5ecf29805ae, entries=150, sequenceid=197, filesize=11.9 K 2024-11-21T21:26:38,675 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2ddd2a6cb469a423cd30254b88184ba0 in 1287ms, sequenceid=197, compaction requested=false 2024-11-21T21:26:38,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:38,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:38,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-21T21:26:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-21T21:26:38,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T21:26:38,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2050 sec 2024-11-21T21:26:38,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.2090 sec 2024-11-21T21:26:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:38,702 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:38,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T21:26:38,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121845175dfefc8448984ec16cba63a6650_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224398702/Put/seqid=0 2024-11-21T21:26:38,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742450_1626 (size=12304) 2024-11-21T21:26:38,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224458721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224458726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224458727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224458830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224458840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224458840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224459039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224459050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224459050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:39,128 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121845175dfefc8448984ec16cba63a6650_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121845175dfefc8448984ec16cba63a6650_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:39,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/2269c54d19f3472bb5be522f37f7336c, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:39,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/2269c54d19f3472bb5be522f37f7336c is 175, key is test_row_0/A:col10/1732224398702/Put/seqid=0 2024-11-21T21:26:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742451_1627 (size=31105) 2024-11-21T21:26:39,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224459349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224459359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224459359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/2269c54d19f3472bb5be522f37f7336c 2024-11-21T21:26:39,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/a2f70042b25140a8bd802926486b2602 is 50, key is test_row_0/B:col10/1732224398702/Put/seqid=0 2024-11-21T21:26:39,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742452_1628 (size=12151) 2024-11-21T21:26:39,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224459860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224459869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224459869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:39,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/a2f70042b25140a8bd802926486b2602 2024-11-21T21:26:39,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4e8b31a354a64d908b733a326d5e6c45 is 50, key is test_row_0/C:col10/1732224398702/Put/seqid=0 2024-11-21T21:26:39,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742453_1629 (size=12151) 2024-11-21T21:26:40,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4e8b31a354a64d908b733a326d5e6c45 2024-11-21T21:26:40,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/2269c54d19f3472bb5be522f37f7336c as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/2269c54d19f3472bb5be522f37f7336c 2024-11-21T21:26:40,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/2269c54d19f3472bb5be522f37f7336c, entries=150, sequenceid=218, filesize=30.4 K 2024-11-21T21:26:40,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/a2f70042b25140a8bd802926486b2602 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/a2f70042b25140a8bd802926486b2602 2024-11-21T21:26:40,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/a2f70042b25140a8bd802926486b2602, entries=150, sequenceid=218, filesize=11.9 K 2024-11-21T21:26:40,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4e8b31a354a64d908b733a326d5e6c45 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4e8b31a354a64d908b733a326d5e6c45 2024-11-21T21:26:40,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4e8b31a354a64d908b733a326d5e6c45, entries=150, sequenceid=218, filesize=11.9 K 2024-11-21T21:26:40,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2ddd2a6cb469a423cd30254b88184ba0 in 1665ms, sequenceid=218, compaction requested=true 2024-11-21T21:26:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:40,367 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:40,367 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:40,368 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:40,368 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:40,368 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,368 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/19ea40b5b1d5480bb3fa11c185b18244, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/0248bea07a814f22b33402f1be93a08e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/2269c54d19f3472bb5be522f37f7336c] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=91.5 K 2024-11-21T21:26:40,368 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,368 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/19ea40b5b1d5480bb3fa11c185b18244, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/0248bea07a814f22b33402f1be93a08e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/2269c54d19f3472bb5be522f37f7336c] 2024-11-21T21:26:40,369 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:40,369 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:40,369 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,369 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/041d75aef9e844acb359a412f9e9f54c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/23fffe7ce89a42c585b757a303356036, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/a2f70042b25140a8bd802926486b2602] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=36.0 K 2024-11-21T21:26:40,369 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19ea40b5b1d5480bb3fa11c185b18244, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732224395229 2024-11-21T21:26:40,370 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 041d75aef9e844acb359a412f9e9f54c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732224395229 2024-11-21T21:26:40,370 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0248bea07a814f22b33402f1be93a08e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732224396397 2024-11-21T21:26:40,370 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fffe7ce89a42c585b757a303356036, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732224396397 2024-11-21T21:26:40,370 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2269c54d19f3472bb5be522f37f7336c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732224397558 2024-11-21T21:26:40,370 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f70042b25140a8bd802926486b2602, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732224397558 2024-11-21T21:26:40,376 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:40,377 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:40,377 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/adebe201cffa43908473d303df81a242 is 50, key is test_row_0/B:col10/1732224398702/Put/seqid=0 2024-11-21T21:26:40,399 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a135990e6bff42b2af0a37635cf56867_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:40,401 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a135990e6bff42b2af0a37635cf56867_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:40,402 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a135990e6bff42b2af0a37635cf56867_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:40,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742454_1630 (size=12663) 2024-11-21T21:26:40,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742455_1631 (size=4469) 2024-11-21T21:26:40,410 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#532 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:40,410 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/ed63b01366b744afa4a5bee36e0e8392 is 175, key is test_row_0/A:col10/1732224398702/Put/seqid=0 2024-11-21T21:26:40,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742456_1632 (size=31617) 2024-11-21T21:26:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:40,481 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:40,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T21:26:40,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212914bb875bab48f0adc0f259aee169cf_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224400480/Put/seqid=0 2024-11-21T21:26:40,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742457_1633 (size=14794) 2024-11-21T21:26:40,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:40,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212914bb875bab48f0adc0f259aee169cf_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212914bb875bab48f0adc0f259aee169cf_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:40,545 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/628e32de24b646ebadecc858f1b79a63, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:40,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/628e32de24b646ebadecc858f1b79a63 is 175, key is test_row_0/A:col10/1732224400480/Put/seqid=0 2024-11-21T21:26:40,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742458_1634 (size=39749) 2024-11-21T21:26:40,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/628e32de24b646ebadecc858f1b79a63 2024-11-21T21:26:40,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9add532ab327461eaa36cc548973bb7d is 50, key is test_row_0/B:col10/1732224400480/Put/seqid=0 2024-11-21T21:26:40,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742459_1635 (size=12151) 2024-11-21T21:26:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224460591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224460596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:26:40,640 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:40,641 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-21T21:26:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:26:40,643 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:40,644 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:40,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:40,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224460699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224460710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:26:40,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:26:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:40,806 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/adebe201cffa43908473d303df81a242 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/adebe201cffa43908473d303df81a242 2024-11-21T21:26:40,809 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into adebe201cffa43908473d303df81a242(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:40,809 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:40,809 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=13, startTime=1732224400367; duration=0sec 2024-11-21T21:26:40,810 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:40,810 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:40,810 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:40,810 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:40,810 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:40,811 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,811 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ca118fe4dd7a4ab0bc3c3e98440b6db3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7625cb26c9264eb5a84fd5ecf29805ae, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4e8b31a354a64d908b733a326d5e6c45] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=36.0 K 2024-11-21T21:26:40,811 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ca118fe4dd7a4ab0bc3c3e98440b6db3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732224395229 2024-11-21T21:26:40,811 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7625cb26c9264eb5a84fd5ecf29805ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732224396397 2024-11-21T21:26:40,811 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e8b31a354a64d908b733a326d5e6c45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732224397558 2024-11-21T21:26:40,818 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:40,818 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/2ab2e9a76d2c44b5b750bd22afa92cde is 50, key is test_row_0/C:col10/1732224398702/Put/seqid=0 2024-11-21T21:26:40,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742460_1636 (size=12663) 2024-11-21T21:26:40,831 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/ed63b01366b744afa4a5bee36e0e8392 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/ed63b01366b744afa4a5bee36e0e8392 2024-11-21T21:26:40,836 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into ed63b01366b744afa4a5bee36e0e8392(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:40,836 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:40,836 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=13, startTime=1732224400367; duration=0sec 2024-11-21T21:26:40,836 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:40,836 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:40,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224460869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,873 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224460879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,883 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224460890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,894 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224460910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224460919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:40,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:26:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:40,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:40,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:26:40,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9add532ab327461eaa36cc548973bb7d 2024-11-21T21:26:40,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/28c0a2b95efa4d69940b12335cde21a9 is 50, key is test_row_0/C:col10/1732224400480/Put/seqid=0 2024-11-21T21:26:40,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742461_1637 (size=12151) 2024-11-21T21:26:41,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/28c0a2b95efa4d69940b12335cde21a9 2024-11-21T21:26:41,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/628e32de24b646ebadecc858f1b79a63 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/628e32de24b646ebadecc858f1b79a63 2024-11-21T21:26:41,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/628e32de24b646ebadecc858f1b79a63, entries=200, sequenceid=237, filesize=38.8 K 2024-11-21T21:26:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/9add532ab327461eaa36cc548973bb7d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9add532ab327461eaa36cc548973bb7d 2024-11-21T21:26:41,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9add532ab327461eaa36cc548973bb7d, entries=150, sequenceid=237, filesize=11.9 K 2024-11-21T21:26:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/28c0a2b95efa4d69940b12335cde21a9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/28c0a2b95efa4d69940b12335cde21a9 2024-11-21T21:26:41,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/28c0a2b95efa4d69940b12335cde21a9, entries=150, sequenceid=237, filesize=11.9 K 2024-11-21T21:26:41,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2ddd2a6cb469a423cd30254b88184ba0 in 537ms, sequenceid=237, compaction requested=false 2024-11-21T21:26:41,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:41,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:26:41,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:41,103 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T21:26:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211f2ffafe7efa4fe5b152dc3b61730d83_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224400594/Put/seqid=0 2024-11-21T21:26:41,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742462_1638 (size=12304) 2024-11-21T21:26:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:41,231 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/2ab2e9a76d2c44b5b750bd22afa92cde as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2ab2e9a76d2c44b5b750bd22afa92cde 2024-11-21T21:26:41,235 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into 2ab2e9a76d2c44b5b750bd22afa92cde(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:41,235 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:41,235 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=13, startTime=1732224400367; duration=0sec 2024-11-21T21:26:41,235 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:41,235 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224461259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224461260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:26:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224461369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224461369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:41,534 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211f2ffafe7efa4fe5b152dc3b61730d83_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211f2ffafe7efa4fe5b152dc3b61730d83_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:41,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9f877872a1b34b5b9c03c1ce308d51ff, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9f877872a1b34b5b9c03c1ce308d51ff is 175, key is test_row_0/A:col10/1732224400594/Put/seqid=0 2024-11-21T21:26:41,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742463_1639 (size=31105) 2024-11-21T21:26:41,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224461579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224461579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:26:41,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224461890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224461890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:41,939 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9f877872a1b34b5b9c03c1ce308d51ff 2024-11-21T21:26:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/8e265bb2035b4a95a5404bf63fb8895e is 50, key is test_row_0/B:col10/1732224400594/Put/seqid=0 2024-11-21T21:26:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742464_1640 (size=12151) 2024-11-21T21:26:41,962 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/8e265bb2035b4a95a5404bf63fb8895e 2024-11-21T21:26:41,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/1da1bdb79500450290fc631761ef615f is 50, key is test_row_0/C:col10/1732224400594/Put/seqid=0 2024-11-21T21:26:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742465_1641 (size=12151) 2024-11-21T21:26:41,982 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/1da1bdb79500450290fc631761ef615f 2024-11-21T21:26:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9f877872a1b34b5b9c03c1ce308d51ff as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9f877872a1b34b5b9c03c1ce308d51ff 2024-11-21T21:26:41,988 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9f877872a1b34b5b9c03c1ce308d51ff, entries=150, sequenceid=256, filesize=30.4 K 2024-11-21T21:26:41,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/8e265bb2035b4a95a5404bf63fb8895e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/8e265bb2035b4a95a5404bf63fb8895e 2024-11-21T21:26:41,991 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/8e265bb2035b4a95a5404bf63fb8895e, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T21:26:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/1da1bdb79500450290fc631761ef615f as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/1da1bdb79500450290fc631761ef615f 2024-11-21T21:26:41,995 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/1da1bdb79500450290fc631761ef615f, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T21:26:41,995 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2ddd2a6cb469a423cd30254b88184ba0 in 892ms, sequenceid=256, compaction requested=true 2024-11-21T21:26:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:41,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-21T21:26:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-21T21:26:41,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-21T21:26:41,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3530 sec 2024-11-21T21:26:42,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.3580 sec 2024-11-21T21:26:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:42,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T21:26:42,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121139b769985f742deb68baf3e5ce5af3f_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224401259/Put/seqid=0 2024-11-21T21:26:42,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742466_1642 (size=14994) 2024-11-21T21:26:42,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:42,414 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121139b769985f742deb68baf3e5ce5af3f_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121139b769985f742deb68baf3e5ce5af3f_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:42,415 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9bb9de2502694a54a35cae1ed485edea, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:42,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9bb9de2502694a54a35cae1ed485edea is 175, key is test_row_0/A:col10/1732224401259/Put/seqid=0 2024-11-21T21:26:42,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742467_1643 (size=39949) 2024-11-21T21:26:42,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9bb9de2502694a54a35cae1ed485edea 2024-11-21T21:26:42,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/4ea02f2c66444e2eb4eb0b3c161a8b27 is 50, key is test_row_0/B:col10/1732224401259/Put/seqid=0 2024-11-21T21:26:42,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742468_1644 (size=12301) 2024-11-21T21:26:42,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224462433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224462433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224462540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224462540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224462750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224462750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:26:42,799 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:42,800 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-21T21:26:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:26:42,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:42,802 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:42,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:42,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/4ea02f2c66444e2eb4eb0b3c161a8b27 2024-11-21T21:26:42,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/2f15db06995b4c07a357e29fd4634da1 is 50, key is test_row_0/C:col10/1732224401259/Put/seqid=0 2024-11-21T21:26:42,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742469_1645 (size=12301) 2024-11-21T21:26:42,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224462880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,883 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_0' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224462899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,904 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_2' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:26:42,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224462919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:42,922 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,34123,1732224229084 for 'test_row_1' in 2ddd2a6cb469a423cd30254b88184ba0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:26:42,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:42,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:42,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T21:26:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:42,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:43,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224463059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224463060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:43,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T21:26:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:43,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:26:43,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/2f15db06995b4c07a357e29fd4634da1 2024-11-21T21:26:43,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9bb9de2502694a54a35cae1ed485edea as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9bb9de2502694a54a35cae1ed485edea 2024-11-21T21:26:43,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9bb9de2502694a54a35cae1ed485edea, entries=200, sequenceid=277, filesize=39.0 K 2024-11-21T21:26:43,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/4ea02f2c66444e2eb4eb0b3c161a8b27 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/4ea02f2c66444e2eb4eb0b3c161a8b27 2024-11-21T21:26:43,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/4ea02f2c66444e2eb4eb0b3c161a8b27, entries=150, sequenceid=277, filesize=12.0 K 2024-11-21T21:26:43,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/2f15db06995b4c07a357e29fd4634da1 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2f15db06995b4c07a357e29fd4634da1 2024-11-21T21:26:43,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2f15db06995b4c07a357e29fd4634da1, entries=150, sequenceid=277, filesize=12.0 K 2024-11-21T21:26:43,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2ddd2a6cb469a423cd30254b88184ba0 in 851ms, sequenceid=277, compaction requested=true 2024-11-21T21:26:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:43,253 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:43,253 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:43,254 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142420 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:43,254 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:43,254 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:43,254 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:43,254 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:43,254 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:43,254 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/adebe201cffa43908473d303df81a242, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9add532ab327461eaa36cc548973bb7d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/8e265bb2035b4a95a5404bf63fb8895e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/4ea02f2c66444e2eb4eb0b3c161a8b27] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=48.1 K 2024-11-21T21:26:43,254 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/ed63b01366b744afa4a5bee36e0e8392, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/628e32de24b646ebadecc858f1b79a63, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9f877872a1b34b5b9c03c1ce308d51ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9bb9de2502694a54a35cae1ed485edea] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=139.1 K 2024-11-21T21:26:43,255 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:43,255 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/ed63b01366b744afa4a5bee36e0e8392, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/628e32de24b646ebadecc858f1b79a63, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9f877872a1b34b5b9c03c1ce308d51ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9bb9de2502694a54a35cae1ed485edea] 2024-11-21T21:26:43,255 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed63b01366b744afa4a5bee36e0e8392, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732224397558 2024-11-21T21:26:43,255 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting adebe201cffa43908473d303df81a242, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732224397558 2024-11-21T21:26:43,255 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9add532ab327461eaa36cc548973bb7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732224398725 2024-11-21T21:26:43,255 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 628e32de24b646ebadecc858f1b79a63, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732224398720 2024-11-21T21:26:43,255 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e265bb2035b4a95a5404bf63fb8895e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224400584 2024-11-21T21:26:43,255 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f877872a1b34b5b9c03c1ce308d51ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224400584 2024-11-21T21:26:43,256 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea02f2c66444e2eb4eb0b3c161a8b27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732224401254 2024-11-21T21:26:43,256 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bb9de2502694a54a35cae1ed485edea, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732224401254 2024-11-21T21:26:43,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T21:26:43,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:43,261 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T21:26:43,263 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#543 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:43,264 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/59a2c9159c9845ab97d28b6836fe0e7b is 50, key is test_row_0/B:col10/1732224401259/Put/seqid=0 2024-11-21T21:26:43,269 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:43,273 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112195d46a83dc2747fba76e960c67f4bd42_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:43,275 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112195d46a83dc2747fba76e960c67f4bd42_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:43,275 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112195d46a83dc2747fba76e960c67f4bd42_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:43,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742470_1646 (size=12949) 2024-11-21T21:26:43,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121965b18442b7347feb79f6429b774b08b_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224402411/Put/seqid=0 2024-11-21T21:26:43,283 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/59a2c9159c9845ab97d28b6836fe0e7b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/59a2c9159c9845ab97d28b6836fe0e7b 2024-11-21T21:26:43,287 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into 59a2c9159c9845ab97d28b6836fe0e7b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:43,287 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:43,287 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=12, startTime=1732224403253; duration=0sec 2024-11-21T21:26:43,287 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:43,287 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:43,287 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:43,288 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:43,288 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:43,288 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:43,288 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2ab2e9a76d2c44b5b750bd22afa92cde, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/28c0a2b95efa4d69940b12335cde21a9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/1da1bdb79500450290fc631761ef615f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2f15db06995b4c07a357e29fd4634da1] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=48.1 K 2024-11-21T21:26:43,288 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab2e9a76d2c44b5b750bd22afa92cde, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732224397558 2024-11-21T21:26:43,289 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c0a2b95efa4d69940b12335cde21a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732224398725 2024-11-21T21:26:43,289 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1da1bdb79500450290fc631761ef615f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732224400584 2024-11-21T21:26:43,290 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f15db06995b4c07a357e29fd4634da1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732224401254 2024-11-21T21:26:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742471_1647 (size=4469) 2024-11-21T21:26:43,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742472_1648 (size=12454) 2024-11-21T21:26:43,314 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#546 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:43,315 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4b7e48e5785745a59b815871916aac90 is 50, key is test_row_0/C:col10/1732224401259/Put/seqid=0 2024-11-21T21:26:43,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742473_1649 (size=12949) 2024-11-21T21:26:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:26:43,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:43,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224463628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224463629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:43,703 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#544 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:43,703 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/b51e320f00b543189ff3ccebf1548b47 is 175, key is test_row_0/A:col10/1732224401259/Put/seqid=0 2024-11-21T21:26:43,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742474_1650 (size=31903) 2024-11-21T21:26:43,710 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/b51e320f00b543189ff3ccebf1548b47 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b51e320f00b543189ff3ccebf1548b47 2024-11-21T21:26:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:43,715 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into b51e320f00b543189ff3ccebf1548b47(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:43,715 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:43,715 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=12, startTime=1732224403253; duration=0sec 2024-11-21T21:26:43,715 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:43,715 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:43,718 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121965b18442b7347feb79f6429b774b08b_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121965b18442b7347feb79f6429b774b08b_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a1e1c19f4c654de2b199877773a5cf8e, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a1e1c19f4c654de2b199877773a5cf8e is 175, key is test_row_0/A:col10/1732224402411/Put/seqid=0 2024-11-21T21:26:43,724 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4b7e48e5785745a59b815871916aac90 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4b7e48e5785745a59b815871916aac90 2024-11-21T21:26:43,728 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into 4b7e48e5785745a59b815871916aac90(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:43,729 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:43,729 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=12, startTime=1732224403253; duration=0sec 2024-11-21T21:26:43,729 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:43,729 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:43,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742475_1651 (size=31255) 2024-11-21T21:26:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224463739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224463739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:26:43,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224463949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224463949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:44,140 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a1e1c19f4c654de2b199877773a5cf8e 2024-11-21T21:26:44,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/577c33f7fbc94c67864c837b9f1a66d6 is 50, key is test_row_0/B:col10/1732224402411/Put/seqid=0 2024-11-21T21:26:44,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742476_1652 (size=12301) 2024-11-21T21:26:44,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224464260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224464260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:44,549 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/577c33f7fbc94c67864c837b9f1a66d6 2024-11-21T21:26:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/963f306f91f44091b228d9ef121e0231 is 50, key is test_row_0/C:col10/1732224402411/Put/seqid=0 2024-11-21T21:26:44,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742477_1653 (size=12301) 2024-11-21T21:26:44,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224464769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:44,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224464769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:26:44,973 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/963f306f91f44091b228d9ef121e0231 2024-11-21T21:26:44,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a1e1c19f4c654de2b199877773a5cf8e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a1e1c19f4c654de2b199877773a5cf8e 2024-11-21T21:26:44,978 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a1e1c19f4c654de2b199877773a5cf8e, entries=150, sequenceid=293, filesize=30.5 K 2024-11-21T21:26:44,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/577c33f7fbc94c67864c837b9f1a66d6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/577c33f7fbc94c67864c837b9f1a66d6 2024-11-21T21:26:44,981 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/577c33f7fbc94c67864c837b9f1a66d6, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T21:26:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/963f306f91f44091b228d9ef121e0231 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/963f306f91f44091b228d9ef121e0231 2024-11-21T21:26:44,984 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/963f306f91f44091b228d9ef121e0231, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T21:26:44,985 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2ddd2a6cb469a423cd30254b88184ba0 in 1724ms, sequenceid=293, compaction requested=false 2024-11-21T21:26:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-21T21:26:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-21T21:26:44,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T21:26:44,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-11-21T21:26:44,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.1880 sec 2024-11-21T21:26:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:45,782 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:45,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:26:45,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112112ff7f9b1c2647db9ea9657326a71e21_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224405781/Put/seqid=0 2024-11-21T21:26:45,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742478_1654 (size=17534) 2024-11-21T21:26:45,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:45,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112112ff7f9b1c2647db9ea9657326a71e21_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112112ff7f9b1c2647db9ea9657326a71e21_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:45,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a6e625f37af54393b7a33fe6130aa9ff, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:45,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a6e625f37af54393b7a33fe6130aa9ff is 175, key is test_row_0/A:col10/1732224405781/Put/seqid=0 2024-11-21T21:26:45,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742479_1655 (size=48639) 2024-11-21T21:26:45,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224465805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:45,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224465809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:45,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224465919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:45,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224465919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224466130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224466130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,198 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a6e625f37af54393b7a33fe6130aa9ff 2024-11-21T21:26:46,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/bc1aa170e92b40b98ce75e4d8a7180fa is 50, key is test_row_0/B:col10/1732224405781/Put/seqid=0 2024-11-21T21:26:46,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742480_1656 (size=12301) 2024-11-21T21:26:46,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/bc1aa170e92b40b98ce75e4d8a7180fa 2024-11-21T21:26:46,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/ad408c63724341aeb72e81256161e1fe is 50, key is test_row_0/C:col10/1732224405781/Put/seqid=0 2024-11-21T21:26:46,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742481_1657 (size=12301) 2024-11-21T21:26:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224466439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224466439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/ad408c63724341aeb72e81256161e1fe 2024-11-21T21:26:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/a6e625f37af54393b7a33fe6130aa9ff as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a6e625f37af54393b7a33fe6130aa9ff 2024-11-21T21:26:46,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a6e625f37af54393b7a33fe6130aa9ff, entries=250, sequenceid=318, filesize=47.5 K 2024-11-21T21:26:46,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/bc1aa170e92b40b98ce75e4d8a7180fa as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/bc1aa170e92b40b98ce75e4d8a7180fa 2024-11-21T21:26:46,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/bc1aa170e92b40b98ce75e4d8a7180fa, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T21:26:46,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/ad408c63724341aeb72e81256161e1fe as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ad408c63724341aeb72e81256161e1fe 2024-11-21T21:26:46,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ad408c63724341aeb72e81256161e1fe, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T21:26:46,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2ddd2a6cb469a423cd30254b88184ba0 in 842ms, sequenceid=318, compaction requested=true 2024-11-21T21:26:46,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:46,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:46,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:46,625 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:46,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:46,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:46,625 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:46,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:46,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:46,625 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:46,625 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:46,625 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:46,625 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:46,625 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:46,625 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:46,626 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b51e320f00b543189ff3ccebf1548b47, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a1e1c19f4c654de2b199877773a5cf8e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a6e625f37af54393b7a33fe6130aa9ff] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=109.2 K 2024-11-21T21:26:46,626 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/59a2c9159c9845ab97d28b6836fe0e7b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/577c33f7fbc94c67864c837b9f1a66d6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/bc1aa170e92b40b98ce75e4d8a7180fa] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=36.7 K 2024-11-21T21:26:46,626 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:46,626 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b51e320f00b543189ff3ccebf1548b47, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a1e1c19f4c654de2b199877773a5cf8e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a6e625f37af54393b7a33fe6130aa9ff] 2024-11-21T21:26:46,626 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a2c9159c9845ab97d28b6836fe0e7b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732224401254 2024-11-21T21:26:46,626 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51e320f00b543189ff3ccebf1548b47, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732224401254 2024-11-21T21:26:46,626 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e1c19f4c654de2b199877773a5cf8e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224402411 2024-11-21T21:26:46,626 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 577c33f7fbc94c67864c837b9f1a66d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224402411 2024-11-21T21:26:46,626 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6e625f37af54393b7a33fe6130aa9ff, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732224403613 2024-11-21T21:26:46,626 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1aa170e92b40b98ce75e4d8a7180fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732224403613 2024-11-21T21:26:46,631 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#552 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:46,631 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/6c12889b5436441faf1970e3d20f27bc is 50, key is test_row_0/B:col10/1732224405781/Put/seqid=0 2024-11-21T21:26:46,632 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:46,633 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211344756f68264d6499a096d636c7e041_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:46,635 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211344756f68264d6499a096d636c7e041_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:46,635 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211344756f68264d6499a096d636c7e041_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742482_1658 (size=13051) 2024-11-21T21:26:46,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742483_1659 (size=4469) 2024-11-21T21:26:46,640 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/6c12889b5436441faf1970e3d20f27bc as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/6c12889b5436441faf1970e3d20f27bc 2024-11-21T21:26:46,643 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into 6c12889b5436441faf1970e3d20f27bc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:46,643 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:46,643 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=13, startTime=1732224406625; duration=0sec 2024-11-21T21:26:46,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:46,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:46,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:26:46,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:26:46,644 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:46,644 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:46,644 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4b7e48e5785745a59b815871916aac90, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/963f306f91f44091b228d9ef121e0231, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ad408c63724341aeb72e81256161e1fe] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=36.7 K 2024-11-21T21:26:46,645 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b7e48e5785745a59b815871916aac90, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732224401254 2024-11-21T21:26:46,645 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 963f306f91f44091b228d9ef121e0231, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732224402411 2024-11-21T21:26:46,645 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting ad408c63724341aeb72e81256161e1fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732224403613 2024-11-21T21:26:46,649 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#554 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:46,650 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/29acb9f7973e4513a0fb8c33dc3f1084 is 50, key is test_row_0/C:col10/1732224405781/Put/seqid=0 2024-11-21T21:26:46,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742484_1660 (size=13051) 2024-11-21T21:26:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:46,922 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:46,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T21:26:46,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121601834dce7154e63b0390dc8b289c115_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224406921/Put/seqid=0 2024-11-21T21:26:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742485_1661 (size=12454) 2024-11-21T21:26:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:26:46,960 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:46,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:26:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-21T21:26:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:26:46,962 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:26:46,962 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:26:46,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:26:46,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224466986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224466986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224466991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224466992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224466993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,039 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#553 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:47,040 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/fe571b7ba8c04676a49f792cefdbad14 is 175, key is test_row_0/A:col10/1732224405781/Put/seqid=0 2024-11-21T21:26:47,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742486_1662 (size=32005) 2024-11-21T21:26:47,056 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/29acb9f7973e4513a0fb8c33dc3f1084 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/29acb9f7973e4513a0fb8c33dc3f1084 2024-11-21T21:26:47,058 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into 29acb9f7973e4513a0fb8c33dc3f1084(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:47,058 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:47,059 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=13, startTime=1732224406625; duration=0sec 2024-11-21T21:26:47,059 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:47,059 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:26:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224467099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224467099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224467099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224467099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224467099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,140 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:26:47,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:26:47,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224467309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224467309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224467309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224467309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224467310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:47,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121601834dce7154e63b0390dc8b289c115_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121601834dce7154e63b0390dc8b289c115_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:47,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/741fa41ded68450c9726e3ae72020f70, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:47,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/741fa41ded68450c9726e3ae72020f70 is 175, key is test_row_0/A:col10/1732224406921/Put/seqid=0 2024-11-21T21:26:47,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742487_1663 (size=31255) 2024-11-21T21:26:47,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/741fa41ded68450c9726e3ae72020f70 2024-11-21T21:26:47,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/126f9241b2d240539aaeefeb8c2433f6 is 50, key is test_row_0/B:col10/1732224406921/Put/seqid=0 2024-11-21T21:26:47,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742488_1664 (size=12301) 2024-11-21T21:26:47,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,446 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/fe571b7ba8c04676a49f792cefdbad14 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fe571b7ba8c04676a49f792cefdbad14 2024-11-21T21:26:47,449 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into fe571b7ba8c04676a49f792cefdbad14(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:47,450 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:47,450 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=13, startTime=1732224406625; duration=0sec 2024-11-21T21:26:47,450 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:47,450 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:47,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:26:47,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224467619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224467619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224467619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224467619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224467619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:47,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/126f9241b2d240539aaeefeb8c2433f6 2024-11-21T21:26:47,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/0c8a1cebd2454ec2a28a38a58f7f4c70 is 50, key is test_row_0/C:col10/1732224406921/Put/seqid=0 2024-11-21T21:26:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742489_1665 (size=12301) 2024-11-21T21:26:47,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:47,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:48,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:48,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:48,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:48,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:48,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:48,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:26:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:26:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224468140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224468141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224468141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:48,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224468141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224468141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:48,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/0c8a1cebd2454ec2a28a38a58f7f4c70 2024-11-21T21:26:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/741fa41ded68450c9726e3ae72020f70 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/741fa41ded68450c9726e3ae72020f70 2024-11-21T21:26:48,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/741fa41ded68450c9726e3ae72020f70, entries=150, sequenceid=334, filesize=30.5 K 2024-11-21T21:26:48,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/126f9241b2d240539aaeefeb8c2433f6 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/126f9241b2d240539aaeefeb8c2433f6 2024-11-21T21:26:48,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/126f9241b2d240539aaeefeb8c2433f6, entries=150, sequenceid=334, filesize=12.0 K 2024-11-21T21:26:48,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/0c8a1cebd2454ec2a28a38a58f7f4c70 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0c8a1cebd2454ec2a28a38a58f7f4c70 2024-11-21T21:26:48,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0c8a1cebd2454ec2a28a38a58f7f4c70, entries=150, sequenceid=334, filesize=12.0 K 2024-11-21T21:26:48,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2ddd2a6cb469a423cd30254b88184ba0 in 1250ms, sequenceid=334, compaction requested=false 2024-11-21T21:26:48,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:48,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:26:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:48,185 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:26:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216049a7a0783f46898506508aed516e76_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224406991/Put/seqid=0 2024-11-21T21:26:48,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742490_1666 (size=12454) 2024-11-21T21:26:48,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:48,198 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216049a7a0783f46898506508aed516e76_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216049a7a0783f46898506508aed516e76_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:48,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1518aa12d22b45518806cf10b8aa6de8, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1518aa12d22b45518806cf10b8aa6de8 is 175, key is test_row_0/A:col10/1732224406991/Put/seqid=0 2024-11-21T21:26:48,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742491_1667 (size=31255) 2024-11-21T21:26:48,604 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1518aa12d22b45518806cf10b8aa6de8 2024-11-21T21:26:48,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/460f9fae97034703bd36a6b9620b999d is 50, key is test_row_0/B:col10/1732224406991/Put/seqid=0 2024-11-21T21:26:48,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742492_1668 (size=12301) 2024-11-21T21:26:49,018 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/460f9fae97034703bd36a6b9620b999d 2024-11-21T21:26:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4907b1aa6ff34d3494b9861234710cb5 is 50, key is test_row_0/C:col10/1732224406991/Put/seqid=0 2024-11-21T21:26:49,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742493_1669 (size=12301) 2024-11-21T21:26:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:26:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:49,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. as already flushing 2024-11-21T21:26:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224469164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224469166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224469167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224469168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224469169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224469279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224469279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224469279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224469280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224469280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,442 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4907b1aa6ff34d3494b9861234710cb5 2024-11-21T21:26:49,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1518aa12d22b45518806cf10b8aa6de8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1518aa12d22b45518806cf10b8aa6de8 2024-11-21T21:26:49,448 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1518aa12d22b45518806cf10b8aa6de8, entries=150, sequenceid=357, filesize=30.5 K 2024-11-21T21:26:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/460f9fae97034703bd36a6b9620b999d as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/460f9fae97034703bd36a6b9620b999d 2024-11-21T21:26:49,451 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/460f9fae97034703bd36a6b9620b999d, entries=150, sequenceid=357, filesize=12.0 K 2024-11-21T21:26:49,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/4907b1aa6ff34d3494b9861234710cb5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4907b1aa6ff34d3494b9861234710cb5 2024-11-21T21:26:49,454 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4907b1aa6ff34d3494b9861234710cb5, entries=150, sequenceid=357, filesize=12.0 K 2024-11-21T21:26:49,455 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2ddd2a6cb469a423cd30254b88184ba0 in 1270ms, sequenceid=357, compaction requested=true 2024-11-21T21:26:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-21T21:26:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-21T21:26:49,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T21:26:49,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4940 sec 2024-11-21T21:26:49,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.4980 sec 2024-11-21T21:26:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:49,493 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:49,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T21:26:49,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e5d4758982334674a9b57489540b6252_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224409161/Put/seqid=0 2024-11-21T21:26:49,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742494_1670 (size=12454) 2024-11-21T21:26:49,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:49,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e5d4758982334674a9b57489540b6252_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e5d4758982334674a9b57489540b6252_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:49,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9fd248be68e0484bad89ead55c5e2452, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:49,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9fd248be68e0484bad89ead55c5e2452 is 175, key is test_row_0/A:col10/1732224409161/Put/seqid=0 2024-11-21T21:26:49,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742495_1671 (size=31255) 2024-11-21T21:26:49,509 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9fd248be68e0484bad89ead55c5e2452 2024-11-21T21:26:49,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/04a7fa9f34394088bf2de282fa902aff is 50, key is test_row_0/B:col10/1732224409161/Put/seqid=0 2024-11-21T21:26:49,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224469509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742496_1672 (size=12301) 2024-11-21T21:26:49,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/04a7fa9f34394088bf2de282fa902aff 2024-11-21T21:26:49,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224469517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224469517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/5e5407c8d7804281af047a769e36ab9a is 50, key is test_row_0/C:col10/1732224409161/Put/seqid=0 2024-11-21T21:26:49,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742497_1673 (size=12301) 2024-11-21T21:26:49,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/5e5407c8d7804281af047a769e36ab9a 2024-11-21T21:26:49,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/9fd248be68e0484bad89ead55c5e2452 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9fd248be68e0484bad89ead55c5e2452 2024-11-21T21:26:49,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9fd248be68e0484bad89ead55c5e2452, entries=150, sequenceid=374, filesize=30.5 K 2024-11-21T21:26:49,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/04a7fa9f34394088bf2de282fa902aff as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/04a7fa9f34394088bf2de282fa902aff 2024-11-21T21:26:49,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/04a7fa9f34394088bf2de282fa902aff, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T21:26:49,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/5e5407c8d7804281af047a769e36ab9a as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5e5407c8d7804281af047a769e36ab9a 2024-11-21T21:26:49,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5e5407c8d7804281af047a769e36ab9a, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T21:26:49,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 2ddd2a6cb469a423cd30254b88184ba0 in 48ms, sequenceid=374, compaction requested=true 2024-11-21T21:26:49,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:49,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:49,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:49,542 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:49,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:49,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:49,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:49,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:49,542 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:49,543 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:49,543 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:49,544 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/B is initiating minor compaction (all files) 2024-11-21T21:26:49,544 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/A is initiating minor compaction (all files) 2024-11-21T21:26:49,544 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/A in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:49,544 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/B in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:49,544 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fe571b7ba8c04676a49f792cefdbad14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/741fa41ded68450c9726e3ae72020f70, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1518aa12d22b45518806cf10b8aa6de8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9fd248be68e0484bad89ead55c5e2452] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=122.8 K 2024-11-21T21:26:49,544 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/6c12889b5436441faf1970e3d20f27bc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/126f9241b2d240539aaeefeb8c2433f6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/460f9fae97034703bd36a6b9620b999d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/04a7fa9f34394088bf2de282fa902aff] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=48.8 K 2024-11-21T21:26:49,544 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:49,544 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. files: [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fe571b7ba8c04676a49f792cefdbad14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/741fa41ded68450c9726e3ae72020f70, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1518aa12d22b45518806cf10b8aa6de8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9fd248be68e0484bad89ead55c5e2452] 2024-11-21T21:26:49,544 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c12889b5436441faf1970e3d20f27bc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732224403613 2024-11-21T21:26:49,545 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe571b7ba8c04676a49f792cefdbad14, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732224403613 2024-11-21T21:26:49,545 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 126f9241b2d240539aaeefeb8c2433f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224405804 2024-11-21T21:26:49,545 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 460f9fae97034703bd36a6b9620b999d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732224406955 2024-11-21T21:26:49,545 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 741fa41ded68450c9726e3ae72020f70, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224405804 2024-11-21T21:26:49,546 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 04a7fa9f34394088bf2de282fa902aff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732224409161 2024-11-21T21:26:49,546 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1518aa12d22b45518806cf10b8aa6de8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732224406955 2024-11-21T21:26:49,546 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd248be68e0484bad89ead55c5e2452, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732224409161 2024-11-21T21:26:49,553 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#B#compaction#564 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:49,553 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/3605f0590c9548e79ee750920e9c4978 is 50, key is test_row_0/B:col10/1732224409161/Put/seqid=0 2024-11-21T21:26:49,554 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:49,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742498_1674 (size=13187) 2024-11-21T21:26:49,561 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/3605f0590c9548e79ee750920e9c4978 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3605f0590c9548e79ee750920e9c4978 2024-11-21T21:26:49,565 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121dfa46b8e384d448c9fde9b0b12b7a071_2ddd2a6cb469a423cd30254b88184ba0 store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:49,565 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/B of 2ddd2a6cb469a423cd30254b88184ba0 into 3605f0590c9548e79ee750920e9c4978(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:49,565 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:49,565 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/B, priority=12, startTime=1732224409542; duration=0sec 2024-11-21T21:26:49,565 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:49,565 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:49,565 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:26:49,567 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121dfa46b8e384d448c9fde9b0b12b7a071_2ddd2a6cb469a423cd30254b88184ba0, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:49,567 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dfa46b8e384d448c9fde9b0b12b7a071_2ddd2a6cb469a423cd30254b88184ba0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:49,568 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:26:49,568 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1541): 2ddd2a6cb469a423cd30254b88184ba0/C is initiating minor compaction (all files) 2024-11-21T21:26:49,568 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ddd2a6cb469a423cd30254b88184ba0/C in TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:49,568 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/29acb9f7973e4513a0fb8c33dc3f1084, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0c8a1cebd2454ec2a28a38a58f7f4c70, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4907b1aa6ff34d3494b9861234710cb5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5e5407c8d7804281af047a769e36ab9a] into tmpdir=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp, totalSize=48.8 K 2024-11-21T21:26:49,568 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 29acb9f7973e4513a0fb8c33dc3f1084, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732224403613 2024-11-21T21:26:49,568 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8a1cebd2454ec2a28a38a58f7f4c70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732224405804 2024-11-21T21:26:49,568 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4907b1aa6ff34d3494b9861234710cb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732224406955 2024-11-21T21:26:49,569 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e5407c8d7804281af047a769e36ab9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732224409161 2024-11-21T21:26:49,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742499_1675 (size=4469) 2024-11-21T21:26:49,572 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#A#compaction#565 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:49,573 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/5bfe01fbcd114ae793358db5c0062a89 is 175, key is test_row_0/A:col10/1732224409161/Put/seqid=0 2024-11-21T21:26:49,577 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ddd2a6cb469a423cd30254b88184ba0#C#compaction#566 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:26:49,577 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/f385687051e043b18b7bde6e66a3eb55 is 50, key is test_row_0/C:col10/1732224409161/Put/seqid=0 2024-11-21T21:26:49,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742500_1676 (size=32141) 2024-11-21T21:26:49,583 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/5bfe01fbcd114ae793358db5c0062a89 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/5bfe01fbcd114ae793358db5c0062a89 2024-11-21T21:26:49,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742501_1677 (size=13187) 2024-11-21T21:26:49,587 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/A of 2ddd2a6cb469a423cd30254b88184ba0 into 5bfe01fbcd114ae793358db5c0062a89(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:49,587 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:49,587 INFO [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/A, priority=12, startTime=1732224409542; duration=0sec 2024-11-21T21:26:49,587 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:49,587 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:49,590 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/f385687051e043b18b7bde6e66a3eb55 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/f385687051e043b18b7bde6e66a3eb55 2024-11-21T21:26:49,594 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ddd2a6cb469a423cd30254b88184ba0/C of 2ddd2a6cb469a423cd30254b88184ba0 into f385687051e043b18b7bde6e66a3eb55(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:26:49,594 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:49,594 INFO [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., storeName=2ddd2a6cb469a423cd30254b88184ba0/C, priority=12, startTime=1732224409542; duration=0sec 2024-11-21T21:26:49,594 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:49,594 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:49,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:49,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T21:26:49,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219405141c050c48b19e590c1f513a8620_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224409512/Put/seqid=0 2024-11-21T21:26:49,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742502_1678 (size=14994) 2024-11-21T21:26:49,632 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:49,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219405141c050c48b19e590c1f513a8620_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219405141c050c48b19e590c1f513a8620_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:49,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1830f4c9511d4546953e07450a1f5291, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:49,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1830f4c9511d4546953e07450a1f5291 is 175, key is test_row_0/A:col10/1732224409512/Put/seqid=0 2024-11-21T21:26:49,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742503_1679 (size=39949) 2024-11-21T21:26:49,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224469639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224469639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224469640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224469641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224469641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224469750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224469750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224469750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224469750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224469750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224469960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224469960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224469960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224469960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224469960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1830f4c9511d4546953e07450a1f5291 2024-11-21T21:26:50,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/435035ebbbb44b58b639c93f6f13eea5 is 50, key is test_row_0/B:col10/1732224409512/Put/seqid=0 2024-11-21T21:26:50,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742504_1680 (size=12301) 2024-11-21T21:26:50,166 INFO [Thread-2541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2541. 2024-11-21T21:26:50,166 DEBUG [Thread-2541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:50,166 DEBUG [Thread-2541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,166 DEBUG [Thread-2541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:50,167 INFO [Thread-2537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2537. 2024-11-21T21:26:50,167 DEBUG [Thread-2537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:50,167 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,167 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:50,168 INFO [Thread-2533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2533. 2024-11-21T21:26:50,168 DEBUG [Thread-2533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:50,168 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,168 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:50,169 INFO [Thread-2539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2539. 2024-11-21T21:26:50,169 DEBUG [Thread-2539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:50,169 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,169 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:50,170 INFO [Thread-2535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2535. 2024-11-21T21:26:50,170 DEBUG [Thread-2535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:50,170 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,170 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:50,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224470270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224470269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224470269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224470270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224470270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/435035ebbbb44b58b639c93f6f13eea5 2024-11-21T21:26:50,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/432adfc45ed84f9d8180b049457e9aa9 is 50, key is test_row_0/C:col10/1732224409512/Put/seqid=0 2024-11-21T21:26:50,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742505_1681 (size=12301) 2024-11-21T21:26:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1732224470780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43784 deadline: 1732224470781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43822 deadline: 1732224470781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43836 deadline: 1732224470781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43804 deadline: 1732224470781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 , the old value is region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:26:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., hostname=849dee24a242,34123,1732224229084, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:26:50,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/432adfc45ed84f9d8180b049457e9aa9 2024-11-21T21:26:50,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/1830f4c9511d4546953e07450a1f5291 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1830f4c9511d4546953e07450a1f5291 2024-11-21T21:26:50,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1830f4c9511d4546953e07450a1f5291, entries=200, sequenceid=398, filesize=39.0 K 2024-11-21T21:26:50,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/435035ebbbb44b58b639c93f6f13eea5 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/435035ebbbb44b58b639c93f6f13eea5 2024-11-21T21:26:50,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/435035ebbbb44b58b639c93f6f13eea5, entries=150, sequenceid=398, filesize=12.0 K 2024-11-21T21:26:50,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/432adfc45ed84f9d8180b049457e9aa9 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/432adfc45ed84f9d8180b049457e9aa9 2024-11-21T21:26:50,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/432adfc45ed84f9d8180b049457e9aa9, entries=150, sequenceid=398, filesize=12.0 K 2024-11-21T21:26:50,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2ddd2a6cb469a423cd30254b88184ba0 in 1263ms, sequenceid=398, compaction requested=false 2024-11-21T21:26:50,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:26:51,131 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34123 {}] regionserver.HRegion(8855): Flush requested on 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T21:26:51,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T21:26:51,795 INFO [Thread-2522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2522. 2024-11-21T21:26:51,796 DEBUG [Thread-2522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:51,796 INFO [Thread-2530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2530. 2024-11-21T21:26:51,796 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,796 DEBUG [Thread-2530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:51,796 INFO [Thread-2524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2524. 2024-11-21T21:26:51,796 DEBUG [Thread-2530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,796 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,796 DEBUG [Thread-2524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:51,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:51,796 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,796 DEBUG [Thread-2530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:51,797 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:51,802 INFO [Thread-2526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2526. 2024-11-21T21:26:51,802 INFO [Thread-2528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2528. 2024-11-21T21:26:51,802 DEBUG [Thread-2526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:51,802 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,802 DEBUG [Thread-2528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:26:51,802 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,802 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,802 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:51,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3678 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3557 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3541 2024-11-21T21:26:51,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3682 2024-11-21T21:26:51,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3557 2024-11-21T21:26:51,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:26:51,803 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:26:51,803 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:26:51,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:51,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:51,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:26:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:51,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212cde07a96ba243c197115a0826eccd1f_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_0/A:col10/1732224409639/Put/seqid=0 2024-11-21T21:26:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:26:51,807 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224411806"}]},"ts":"1732224411806"} 2024-11-21T21:26:51,808 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:26:51,808 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:26:51,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742506_1682 (size=12454) 2024-11-21T21:26:51,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:26:51,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, UNASSIGN}] 2024-11-21T21:26:51,810 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, UNASSIGN 2024-11-21T21:26:51,811 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=CLOSING, regionLocation=849dee24a242,34123,1732224229084 2024-11-21T21:26:51,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, UNASSIGN because future has completed 2024-11-21T21:26:51,812 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:26:51,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084}] 2024-11-21T21:26:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:26:51,965 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:51,965 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:26:51,965 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 2ddd2a6cb469a423cd30254b88184ba0, disabling compactions & flushes 2024-11-21T21:26:51,966 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:26:52,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:52,219 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212cde07a96ba243c197115a0826eccd1f_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212cde07a96ba243c197115a0826eccd1f_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:52,220 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/16acd76824484d199782715c0586374b, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:52,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/16acd76824484d199782715c0586374b is 175, key is test_row_0/A:col10/1732224409639/Put/seqid=0 2024-11-21T21:26:52,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742507_1683 (size=31255) 2024-11-21T21:26:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:26:52,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/16acd76824484d199782715c0586374b 2024-11-21T21:26:52,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/77ff22552221441087617ff64d87f824 is 50, key is test_row_0/B:col10/1732224409639/Put/seqid=0 2024-11-21T21:26:52,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742508_1684 (size=12301) 2024-11-21T21:26:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:26:53,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/77ff22552221441087617ff64d87f824 2024-11-21T21:26:53,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/0ed7cb60e7624e8d876a1be23f13c02e is 50, key is test_row_0/C:col10/1732224409639/Put/seqid=0 2024-11-21T21:26:53,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742509_1685 (size=12301) 2024-11-21T21:26:53,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/0ed7cb60e7624e8d876a1be23f13c02e 2024-11-21T21:26:53,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/16acd76824484d199782715c0586374b as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/16acd76824484d199782715c0586374b 2024-11-21T21:26:53,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/16acd76824484d199782715c0586374b, entries=150, sequenceid=416, filesize=30.5 K 2024-11-21T21:26:53,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/77ff22552221441087617ff64d87f824 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/77ff22552221441087617ff64d87f824 2024-11-21T21:26:53,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/77ff22552221441087617ff64d87f824, entries=150, sequenceid=416, filesize=12.0 K 2024-11-21T21:26:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/0ed7cb60e7624e8d876a1be23f13c02e as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0ed7cb60e7624e8d876a1be23f13c02e 2024-11-21T21:26:53,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0ed7cb60e7624e8d876a1be23f13c02e, entries=150, sequenceid=416, filesize=12.0 K 2024-11-21T21:26:53,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 2ddd2a6cb469a423cd30254b88184ba0 in 1680ms, sequenceid=416, compaction requested=true 2024-11-21T21:26:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ddd2a6cb469a423cd30254b88184ba0: 2024-11-21T21:26:53,475 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:26:53,475 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:53,475 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. after waiting 0 ms 2024-11-21T21:26:53,475 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:26:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:26:53,475 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 2ddd2a6cb469a423cd30254b88184ba0 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-21T21:26:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ddd2a6cb469a423cd30254b88184ba0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:26:53,475 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. because compaction request was cancelled 2024-11-21T21:26:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:26:53,475 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:A 2024-11-21T21:26:53,475 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. because compaction request was cancelled 2024-11-21T21:26:53,475 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. because compaction request was cancelled 2024-11-21T21:26:53,475 DEBUG [RS:0;849dee24a242:34123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:B 2024-11-21T21:26:53,475 DEBUG [RS:0;849dee24a242:34123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ddd2a6cb469a423cd30254b88184ba0:C 2024-11-21T21:26:53,481 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cd5f5fb064ab4334aef2b72d86e63e63_2ddd2a6cb469a423cd30254b88184ba0 is 50, key is test_row_1/A:col10/1732224411801/Put/seqid=0 2024-11-21T21:26:53,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742510_1686 (size=9914) 2024-11-21T21:26:53,887 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:26:53,894 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cd5f5fb064ab4334aef2b72d86e63e63_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd5f5fb064ab4334aef2b72d86e63e63_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:53,895 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/8399c643a88842a0b88abbbc5ff9dbae, store: [table=TestAcidGuarantees family=A region=2ddd2a6cb469a423cd30254b88184ba0] 2024-11-21T21:26:53,896 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/8399c643a88842a0b88abbbc5ff9dbae is 175, key is test_row_1/A:col10/1732224411801/Put/seqid=0 2024-11-21T21:26:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742511_1687 (size=22561) 2024-11-21T21:26:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:26:54,302 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/8399c643a88842a0b88abbbc5ff9dbae 2024-11-21T21:26:54,315 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/550fc91574a240a9a48ac31cad3e5e39 is 50, key is test_row_1/B:col10/1732224411801/Put/seqid=0 2024-11-21T21:26:54,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742512_1688 (size=9857) 2024-11-21T21:26:54,720 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/550fc91574a240a9a48ac31cad3e5e39 2024-11-21T21:26:54,733 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/5c51baac00e04322898966b11953a0ac is 50, key is test_row_1/C:col10/1732224411801/Put/seqid=0 2024-11-21T21:26:54,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742513_1689 (size=9857) 2024-11-21T21:26:55,138 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/5c51baac00e04322898966b11953a0ac 2024-11-21T21:26:55,146 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/A/8399c643a88842a0b88abbbc5ff9dbae as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/8399c643a88842a0b88abbbc5ff9dbae 2024-11-21T21:26:55,151 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/8399c643a88842a0b88abbbc5ff9dbae, entries=100, sequenceid=421, filesize=22.0 K 2024-11-21T21:26:55,152 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/B/550fc91574a240a9a48ac31cad3e5e39 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/550fc91574a240a9a48ac31cad3e5e39 2024-11-21T21:26:55,154 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/550fc91574a240a9a48ac31cad3e5e39, entries=100, sequenceid=421, filesize=9.6 K 2024-11-21T21:26:55,155 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/.tmp/C/5c51baac00e04322898966b11953a0ac as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5c51baac00e04322898966b11953a0ac 2024-11-21T21:26:55,157 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5c51baac00e04322898966b11953a0ac, entries=100, sequenceid=421, filesize=9.6 K 2024-11-21T21:26:55,157 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 2ddd2a6cb469a423cd30254b88184ba0 in 1682ms, sequenceid=421, compaction requested=true 2024-11-21T21:26:55,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fc6ce3ed5a6a4b1da3305ba8c04adc64, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f49b4ccf39424b79a186b4fb650fe078, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/33f54441c1f341308830541906f03c78, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/4f337d5742864dffba0639086ec60a27, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/e235dbd786b9456b8b30a327574e96e5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/21a1cf4d7b524b359b2865a75225be2f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/bb8a37ad16fd482cad1f92dd43d9bedd, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/cf6f971dc5ac4c288b1a88d1c37d08c3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b59f246fc0f34a3793b95d8944218ba4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f08413bb32764831aba68b4d67e6d037, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f62de3340a3a4ddc8d350daac7a79545, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/3c0d1de2e15a4d2fa9624762c447cc55, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/19ea40b5b1d5480bb3fa11c185b18244, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/0248bea07a814f22b33402f1be93a08e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/ed63b01366b744afa4a5bee36e0e8392, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/2269c54d19f3472bb5be522f37f7336c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/628e32de24b646ebadecc858f1b79a63, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9f877872a1b34b5b9c03c1ce308d51ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9bb9de2502694a54a35cae1ed485edea, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b51e320f00b543189ff3ccebf1548b47, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a1e1c19f4c654de2b199877773a5cf8e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a6e625f37af54393b7a33fe6130aa9ff, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fe571b7ba8c04676a49f792cefdbad14, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/741fa41ded68450c9726e3ae72020f70, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1518aa12d22b45518806cf10b8aa6de8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9fd248be68e0484bad89ead55c5e2452] to archive 2024-11-21T21:26:55,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:26:55,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fc6ce3ed5a6a4b1da3305ba8c04adc64 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fc6ce3ed5a6a4b1da3305ba8c04adc64 2024-11-21T21:26:55,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f49b4ccf39424b79a186b4fb650fe078 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f49b4ccf39424b79a186b4fb650fe078 2024-11-21T21:26:55,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/33f54441c1f341308830541906f03c78 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/33f54441c1f341308830541906f03c78 2024-11-21T21:26:55,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/4f337d5742864dffba0639086ec60a27 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/4f337d5742864dffba0639086ec60a27 2024-11-21T21:26:55,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/e235dbd786b9456b8b30a327574e96e5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/e235dbd786b9456b8b30a327574e96e5 2024-11-21T21:26:55,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/21a1cf4d7b524b359b2865a75225be2f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/21a1cf4d7b524b359b2865a75225be2f 2024-11-21T21:26:55,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/bb8a37ad16fd482cad1f92dd43d9bedd to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/bb8a37ad16fd482cad1f92dd43d9bedd 2024-11-21T21:26:55,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/cf6f971dc5ac4c288b1a88d1c37d08c3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/cf6f971dc5ac4c288b1a88d1c37d08c3 2024-11-21T21:26:55,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b59f246fc0f34a3793b95d8944218ba4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b59f246fc0f34a3793b95d8944218ba4 2024-11-21T21:26:55,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f08413bb32764831aba68b4d67e6d037 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f08413bb32764831aba68b4d67e6d037 2024-11-21T21:26:55,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f62de3340a3a4ddc8d350daac7a79545 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/f62de3340a3a4ddc8d350daac7a79545 2024-11-21T21:26:55,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/3c0d1de2e15a4d2fa9624762c447cc55 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/3c0d1de2e15a4d2fa9624762c447cc55 2024-11-21T21:26:55,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/19ea40b5b1d5480bb3fa11c185b18244 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/19ea40b5b1d5480bb3fa11c185b18244 2024-11-21T21:26:55,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/0248bea07a814f22b33402f1be93a08e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/0248bea07a814f22b33402f1be93a08e 2024-11-21T21:26:55,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/ed63b01366b744afa4a5bee36e0e8392 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/ed63b01366b744afa4a5bee36e0e8392 2024-11-21T21:26:55,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/2269c54d19f3472bb5be522f37f7336c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/2269c54d19f3472bb5be522f37f7336c 2024-11-21T21:26:55,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/628e32de24b646ebadecc858f1b79a63 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/628e32de24b646ebadecc858f1b79a63 2024-11-21T21:26:55,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9f877872a1b34b5b9c03c1ce308d51ff to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9f877872a1b34b5b9c03c1ce308d51ff 2024-11-21T21:26:55,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9bb9de2502694a54a35cae1ed485edea to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9bb9de2502694a54a35cae1ed485edea 2024-11-21T21:26:55,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b51e320f00b543189ff3ccebf1548b47 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/b51e320f00b543189ff3ccebf1548b47 2024-11-21T21:26:55,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a1e1c19f4c654de2b199877773a5cf8e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a1e1c19f4c654de2b199877773a5cf8e 2024-11-21T21:26:55,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a6e625f37af54393b7a33fe6130aa9ff to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/a6e625f37af54393b7a33fe6130aa9ff 2024-11-21T21:26:55,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fe571b7ba8c04676a49f792cefdbad14 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/fe571b7ba8c04676a49f792cefdbad14 2024-11-21T21:26:55,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/741fa41ded68450c9726e3ae72020f70 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/741fa41ded68450c9726e3ae72020f70 2024-11-21T21:26:55,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1518aa12d22b45518806cf10b8aa6de8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1518aa12d22b45518806cf10b8aa6de8 2024-11-21T21:26:55,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9fd248be68e0484bad89ead55c5e2452 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/9fd248be68e0484bad89ead55c5e2452 2024-11-21T21:26:55,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/93533b9f8f4443a9b8529cb316abbc75, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/e6bad08e07ec4daba9453d5cf9dd2f3e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/aec2edaf5c9c400b9f3c9623ffe571f4, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/913cb16ebd5d49d18a532518b0ae1d76, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9fd987c04a6b47c380205be321e7450d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/172bddae30df47ba8bfea1359ae5ab3f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3fa67289c79a42369691db640b9172e7, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/70caf8a326944a9db960797ddd37b4d5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/06e9b9a179704f39bec85fad0149dc2d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/fdaa0b184bda455aa133224653f6762f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/5ff97aa071b24cada4e2781275728ac1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/041d75aef9e844acb359a412f9e9f54c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9b2403a9c93c4e489f3842375b19fe2c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/23fffe7ce89a42c585b757a303356036, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/adebe201cffa43908473d303df81a242, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/a2f70042b25140a8bd802926486b2602, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9add532ab327461eaa36cc548973bb7d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/8e265bb2035b4a95a5404bf63fb8895e, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/59a2c9159c9845ab97d28b6836fe0e7b, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/4ea02f2c66444e2eb4eb0b3c161a8b27, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/577c33f7fbc94c67864c837b9f1a66d6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/6c12889b5436441faf1970e3d20f27bc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/bc1aa170e92b40b98ce75e4d8a7180fa, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/126f9241b2d240539aaeefeb8c2433f6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/460f9fae97034703bd36a6b9620b999d, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/04a7fa9f34394088bf2de282fa902aff] to archive 2024-11-21T21:26:55,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:26:55,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/93533b9f8f4443a9b8529cb316abbc75 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/93533b9f8f4443a9b8529cb316abbc75 2024-11-21T21:26:55,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/e6bad08e07ec4daba9453d5cf9dd2f3e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/e6bad08e07ec4daba9453d5cf9dd2f3e 2024-11-21T21:26:55,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/aec2edaf5c9c400b9f3c9623ffe571f4 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/aec2edaf5c9c400b9f3c9623ffe571f4 2024-11-21T21:26:55,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/913cb16ebd5d49d18a532518b0ae1d76 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/913cb16ebd5d49d18a532518b0ae1d76 2024-11-21T21:26:55,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9fd987c04a6b47c380205be321e7450d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9fd987c04a6b47c380205be321e7450d 2024-11-21T21:26:55,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/172bddae30df47ba8bfea1359ae5ab3f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/172bddae30df47ba8bfea1359ae5ab3f 2024-11-21T21:26:55,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3fa67289c79a42369691db640b9172e7 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3fa67289c79a42369691db640b9172e7 2024-11-21T21:26:55,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/70caf8a326944a9db960797ddd37b4d5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/70caf8a326944a9db960797ddd37b4d5 2024-11-21T21:26:55,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/06e9b9a179704f39bec85fad0149dc2d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/06e9b9a179704f39bec85fad0149dc2d 2024-11-21T21:26:55,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/fdaa0b184bda455aa133224653f6762f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/fdaa0b184bda455aa133224653f6762f 2024-11-21T21:26:55,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/5ff97aa071b24cada4e2781275728ac1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/5ff97aa071b24cada4e2781275728ac1 2024-11-21T21:26:55,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/041d75aef9e844acb359a412f9e9f54c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/041d75aef9e844acb359a412f9e9f54c 2024-11-21T21:26:55,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9b2403a9c93c4e489f3842375b19fe2c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9b2403a9c93c4e489f3842375b19fe2c 2024-11-21T21:26:55,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/23fffe7ce89a42c585b757a303356036 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/23fffe7ce89a42c585b757a303356036 2024-11-21T21:26:55,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/adebe201cffa43908473d303df81a242 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/adebe201cffa43908473d303df81a242 2024-11-21T21:26:55,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/a2f70042b25140a8bd802926486b2602 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/a2f70042b25140a8bd802926486b2602 2024-11-21T21:26:55,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9add532ab327461eaa36cc548973bb7d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/9add532ab327461eaa36cc548973bb7d 2024-11-21T21:26:55,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/8e265bb2035b4a95a5404bf63fb8895e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/8e265bb2035b4a95a5404bf63fb8895e 2024-11-21T21:26:55,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/59a2c9159c9845ab97d28b6836fe0e7b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/59a2c9159c9845ab97d28b6836fe0e7b 2024-11-21T21:26:55,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/4ea02f2c66444e2eb4eb0b3c161a8b27 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/4ea02f2c66444e2eb4eb0b3c161a8b27 2024-11-21T21:26:55,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/577c33f7fbc94c67864c837b9f1a66d6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/577c33f7fbc94c67864c837b9f1a66d6 2024-11-21T21:26:55,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/6c12889b5436441faf1970e3d20f27bc to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/6c12889b5436441faf1970e3d20f27bc 2024-11-21T21:26:55,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/bc1aa170e92b40b98ce75e4d8a7180fa to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/bc1aa170e92b40b98ce75e4d8a7180fa 2024-11-21T21:26:55,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/126f9241b2d240539aaeefeb8c2433f6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/126f9241b2d240539aaeefeb8c2433f6 2024-11-21T21:26:55,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/460f9fae97034703bd36a6b9620b999d to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/460f9fae97034703bd36a6b9620b999d 2024-11-21T21:26:55,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/04a7fa9f34394088bf2de282fa902aff to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/04a7fa9f34394088bf2de282fa902aff 2024-11-21T21:26:55,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c744cb6aa2f345e9b275f0c6a556969c, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/d070fb1a39da4fb2865f28795fca250f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/34cd0395aabe4c3697b9355749ee45c2, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/48f72f30554149c9977f5f1b641790f6, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b7c63cb82cfd45d1ad253da3fd2c80c8, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/742eadbb44fe4da2b74082849b83917a, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c2ec01eaa8424ea5b737b6a4bdb6f835, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7977b009a37248b2a573f9e46ebc27dc, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/3ff92a6d99144e66851f8959b90914ba, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4318f102fb0d47008dfc3b3c37bef3a0, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c29e4619d1f449829e5358256ce151ba, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ca118fe4dd7a4ab0bc3c3e98440b6db3, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b5ad2e6b129e446e9f0beb610a505173, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7625cb26c9264eb5a84fd5ecf29805ae, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2ab2e9a76d2c44b5b750bd22afa92cde, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4e8b31a354a64d908b733a326d5e6c45, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/28c0a2b95efa4d69940b12335cde21a9, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/1da1bdb79500450290fc631761ef615f, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4b7e48e5785745a59b815871916aac90, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2f15db06995b4c07a357e29fd4634da1, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/963f306f91f44091b228d9ef121e0231, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/29acb9f7973e4513a0fb8c33dc3f1084, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ad408c63724341aeb72e81256161e1fe, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0c8a1cebd2454ec2a28a38a58f7f4c70, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4907b1aa6ff34d3494b9861234710cb5, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5e5407c8d7804281af047a769e36ab9a] to archive 2024-11-21T21:26:55,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:26:55,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c744cb6aa2f345e9b275f0c6a556969c to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c744cb6aa2f345e9b275f0c6a556969c 2024-11-21T21:26:55,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/d070fb1a39da4fb2865f28795fca250f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/d070fb1a39da4fb2865f28795fca250f 2024-11-21T21:26:55,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/34cd0395aabe4c3697b9355749ee45c2 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/34cd0395aabe4c3697b9355749ee45c2 2024-11-21T21:26:55,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/48f72f30554149c9977f5f1b641790f6 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/48f72f30554149c9977f5f1b641790f6 2024-11-21T21:26:55,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b7c63cb82cfd45d1ad253da3fd2c80c8 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b7c63cb82cfd45d1ad253da3fd2c80c8 2024-11-21T21:26:55,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/742eadbb44fe4da2b74082849b83917a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/742eadbb44fe4da2b74082849b83917a 2024-11-21T21:26:55,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c2ec01eaa8424ea5b737b6a4bdb6f835 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c2ec01eaa8424ea5b737b6a4bdb6f835 2024-11-21T21:26:55,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7977b009a37248b2a573f9e46ebc27dc to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7977b009a37248b2a573f9e46ebc27dc 2024-11-21T21:26:55,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/3ff92a6d99144e66851f8959b90914ba to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/3ff92a6d99144e66851f8959b90914ba 2024-11-21T21:26:55,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4318f102fb0d47008dfc3b3c37bef3a0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4318f102fb0d47008dfc3b3c37bef3a0 2024-11-21T21:26:55,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c29e4619d1f449829e5358256ce151ba to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/c29e4619d1f449829e5358256ce151ba 2024-11-21T21:26:55,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ca118fe4dd7a4ab0bc3c3e98440b6db3 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ca118fe4dd7a4ab0bc3c3e98440b6db3 2024-11-21T21:26:55,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b5ad2e6b129e446e9f0beb610a505173 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/b5ad2e6b129e446e9f0beb610a505173 2024-11-21T21:26:55,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7625cb26c9264eb5a84fd5ecf29805ae to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/7625cb26c9264eb5a84fd5ecf29805ae 2024-11-21T21:26:55,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2ab2e9a76d2c44b5b750bd22afa92cde to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2ab2e9a76d2c44b5b750bd22afa92cde 2024-11-21T21:26:55,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4e8b31a354a64d908b733a326d5e6c45 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4e8b31a354a64d908b733a326d5e6c45 2024-11-21T21:26:55,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/28c0a2b95efa4d69940b12335cde21a9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/28c0a2b95efa4d69940b12335cde21a9 2024-11-21T21:26:55,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/1da1bdb79500450290fc631761ef615f to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/1da1bdb79500450290fc631761ef615f 2024-11-21T21:26:55,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4b7e48e5785745a59b815871916aac90 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4b7e48e5785745a59b815871916aac90 2024-11-21T21:26:55,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2f15db06995b4c07a357e29fd4634da1 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/2f15db06995b4c07a357e29fd4634da1 2024-11-21T21:26:55,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/963f306f91f44091b228d9ef121e0231 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/963f306f91f44091b228d9ef121e0231 2024-11-21T21:26:55,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/29acb9f7973e4513a0fb8c33dc3f1084 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/29acb9f7973e4513a0fb8c33dc3f1084 2024-11-21T21:26:55,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ad408c63724341aeb72e81256161e1fe to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/ad408c63724341aeb72e81256161e1fe 2024-11-21T21:26:55,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0c8a1cebd2454ec2a28a38a58f7f4c70 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0c8a1cebd2454ec2a28a38a58f7f4c70 2024-11-21T21:26:55,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4907b1aa6ff34d3494b9861234710cb5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/4907b1aa6ff34d3494b9861234710cb5 2024-11-21T21:26:55,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5e5407c8d7804281af047a769e36ab9a to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5e5407c8d7804281af047a769e36ab9a 2024-11-21T21:26:55,229 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=4 2024-11-21T21:26:55,230 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. 2024-11-21T21:26:55,230 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 2ddd2a6cb469a423cd30254b88184ba0: Waiting for close lock at 1732224411965Running coprocessor pre-close hooks at 1732224411965Disabling compacts and flushes for region at 1732224411965Disabling writes for close at 1732224413475 (+1510 ms)Obtaining lock to block concurrent updates at 1732224413475Preparing flush snapshotting stores in 2ddd2a6cb469a423cd30254b88184ba0 at 1732224413475Finished memstore snapshotting TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732224413476 (+1 ms)Flushing stores of TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0. at 1732224413476Flushing 2ddd2a6cb469a423cd30254b88184ba0/A: creating writer at 1732224413476Flushing mob file 2ddd2a6cb469a423cd30254b88184ba0/A: appending metadata at 1732224413481 (+5 ms)Flushing mob file 2ddd2a6cb469a423cd30254b88184ba0/A: closing flushed file at 1732224413481Flushing 2ddd2a6cb469a423cd30254b88184ba0/A: appending metadata at 1732224413895 (+414 ms)Flushing 2ddd2a6cb469a423cd30254b88184ba0/A: closing flushed file at 1732224413895Flushing 2ddd2a6cb469a423cd30254b88184ba0/B: creating writer at 1732224414312 (+417 ms)Flushing 2ddd2a6cb469a423cd30254b88184ba0/B: appending metadata at 1732224414314 (+2 ms)Flushing 2ddd2a6cb469a423cd30254b88184ba0/B: closing flushed file at 1732224414315 (+1 ms)Flushing 2ddd2a6cb469a423cd30254b88184ba0/C: creating writer at 1732224414729 (+414 ms)Flushing 2ddd2a6cb469a423cd30254b88184ba0/C: appending metadata at 1732224414732 (+3 ms)Flushing 2ddd2a6cb469a423cd30254b88184ba0/C: closing flushed file at 1732224414732Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@662add30: reopening flushed file at 1732224415145 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7819ac25: reopening flushed file at 1732224415151 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5425e27: reopening flushed file at 1732224415154 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 2ddd2a6cb469a423cd30254b88184ba0 in 1682ms, sequenceid=421, compaction requested=true at 1732224415157 (+3 ms)Writing region close event to WAL at 1732224415226 (+69 ms)Running coprocessor post-close hooks at 1732224415230 (+4 ms)Closed at 1732224415230 2024-11-21T21:26:55,231 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:55,232 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=2ddd2a6cb469a423cd30254b88184ba0, regionState=CLOSED 2024-11-21T21:26:55,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 because future has completed 2024-11-21T21:26:55,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-21T21:26:55,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 2ddd2a6cb469a423cd30254b88184ba0, server=849dee24a242,34123,1732224229084 in 3.4210 sec 2024-11-21T21:26:55,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-21T21:26:55,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ddd2a6cb469a423cd30254b88184ba0, UNASSIGN in 3.4240 sec 2024-11-21T21:26:55,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-21T21:26:55,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.4280 sec 2024-11-21T21:26:55,238 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732224415238"}]},"ts":"1732224415238"} 2024-11-21T21:26:55,239 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:26:55,239 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:26:55,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.4360 sec 2024-11-21T21:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:26:55,961 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:55,962 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:26:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:56,010 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:26:56,011 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:56,013 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,014 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C, FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/recovered.edits] 2024-11-21T21:26:56,016 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/16acd76824484d199782715c0586374b to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/16acd76824484d199782715c0586374b 2024-11-21T21:26:56,017 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1830f4c9511d4546953e07450a1f5291 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/1830f4c9511d4546953e07450a1f5291 2024-11-21T21:26:56,018 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/5bfe01fbcd114ae793358db5c0062a89 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/5bfe01fbcd114ae793358db5c0062a89 2024-11-21T21:26:56,019 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/8399c643a88842a0b88abbbc5ff9dbae to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/A/8399c643a88842a0b88abbbc5ff9dbae 2024-11-21T21:26:56,021 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3605f0590c9548e79ee750920e9c4978 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/3605f0590c9548e79ee750920e9c4978 2024-11-21T21:26:56,022 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/435035ebbbb44b58b639c93f6f13eea5 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/435035ebbbb44b58b639c93f6f13eea5 2024-11-21T21:26:56,023 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/550fc91574a240a9a48ac31cad3e5e39 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/550fc91574a240a9a48ac31cad3e5e39 2024-11-21T21:26:56,023 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/77ff22552221441087617ff64d87f824 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/B/77ff22552221441087617ff64d87f824 2024-11-21T21:26:56,025 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0ed7cb60e7624e8d876a1be23f13c02e to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/0ed7cb60e7624e8d876a1be23f13c02e 2024-11-21T21:26:56,026 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/432adfc45ed84f9d8180b049457e9aa9 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/432adfc45ed84f9d8180b049457e9aa9 2024-11-21T21:26:56,027 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5c51baac00e04322898966b11953a0ac to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/5c51baac00e04322898966b11953a0ac 2024-11-21T21:26:56,028 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/f385687051e043b18b7bde6e66a3eb55 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/C/f385687051e043b18b7bde6e66a3eb55 2024-11-21T21:26:56,030 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/recovered.edits/424.seqid to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0/recovered.edits/424.seqid 2024-11-21T21:26:56,031 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/default/TestAcidGuarantees/2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,031 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:26:56,031 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:26:56,032 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T21:26:56,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112111252c972f71458a8f9c964d2af7c2b5_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112111252c972f71458a8f9c964d2af7c2b5_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112112ff7f9b1c2647db9ea9657326a71e21_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112112ff7f9b1c2647db9ea9657326a71e21_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121139b769985f742deb68baf3e5ce5af3f_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121139b769985f742deb68baf3e5ce5af3f_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211f2ffafe7efa4fe5b152dc3b61730d83_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211f2ffafe7efa4fe5b152dc3b61730d83_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121286f58801d29479a8f9fd968bd97a79a_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121286f58801d29479a8f9fd968bd97a79a_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,040 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212914bb875bab48f0adc0f259aee169cf_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212914bb875bab48f0adc0f259aee169cf_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212cde07a96ba243c197115a0826eccd1f_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212cde07a96ba243c197115a0826eccd1f_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214a859354adac46b68f2ba001fb500761_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214a859354adac46b68f2ba001fb500761_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,043 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215287ac8d637b461d8b08095484ff697e_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215287ac8d637b461d8b08095484ff697e_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,045 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215f376574e7f0436499e333aabee75d5b_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215f376574e7f0436499e333aabee75d5b_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,046 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121601834dce7154e63b0390dc8b289c115_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121601834dce7154e63b0390dc8b289c115_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,048 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216049a7a0783f46898506508aed516e76_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216049a7a0783f46898506508aed516e76_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,049 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112172a130b1c0f242a4ad2d5064578c7d6a_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112172a130b1c0f242a4ad2d5064578c7d6a_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,050 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121845175dfefc8448984ec16cba63a6650_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121845175dfefc8448984ec16cba63a6650_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,052 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112187836e5ce2b64c4d88892e654366af03_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112187836e5ce2b64c4d88892e654366af03_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,053 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219405141c050c48b19e590c1f513a8620_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219405141c050c48b19e590c1f513a8620_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,054 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121965b18442b7347feb79f6429b774b08b_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121965b18442b7347feb79f6429b774b08b_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,055 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e66a6e1377c4d70bc292a6b8ab2fe3e_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e66a6e1377c4d70bc292a6b8ab2fe3e_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,057 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd5f5fb064ab4334aef2b72d86e63e63_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd5f5fb064ab4334aef2b72d86e63e63_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,058 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d02114ee3d7c46bcaba75a8e893fb08c_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d02114ee3d7c46bcaba75a8e893fb08c_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,059 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e5d4758982334674a9b57489540b6252_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e5d4758982334674a9b57489540b6252_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,060 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f316fb9760fa4694b61868f6c09c0cd3_2ddd2a6cb469a423cd30254b88184ba0 to hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f316fb9760fa4694b61868f6c09c0cd3_2ddd2a6cb469a423cd30254b88184ba0 2024-11-21T21:26:56,061 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:26:56,062 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:56,064 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:26:56,065 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:26:56,066 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:56,066 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:26:56,066 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732224416066"}]},"ts":"9223372036854775807"} 2024-11-21T21:26:56,067 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:26:56,067 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2ddd2a6cb469a423cd30254b88184ba0, NAME => 'TestAcidGuarantees,,1732224387046.2ddd2a6cb469a423cd30254b88184ba0.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:26:56,067 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:26:56,067 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732224416067"}]},"ts":"9223372036854775807"} 2024-11-21T21:26:56,068 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:26:56,069 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:26:56,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 107 msec 2024-11-21T21:26:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:26:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:26:56,119 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:26:56,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 238) - Thread LEAK? -, OpenFileDescriptor=455 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 500), ProcessCount=11 (was 11), AvailableMemoryMB=11453 (was 11476) 2024-11-21T21:26:56,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T21:26:56,128 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:26:56,128 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:26:56,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:56,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:56,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:26:56,129 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T21:26:56,129 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1133720664, stopped=false 2024-11-21T21:26:56,129 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=849dee24a242,35047,1732224227985 2024-11-21T21:26:56,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:26:56,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:26:56,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:26:56,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:26:56,167 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:26:56,168 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:26:56,168 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:26:56,169 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:26:56,169 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:26:56,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:56,170 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '849dee24a242,34123,1732224229084' ***** 2024-11-21T21:26:56,171 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T21:26:56,171 INFO [RS:0;849dee24a242:34123 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T21:26:56,172 INFO [RS:0;849dee24a242:34123 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T21:26:56,172 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T21:26:56,172 INFO [RS:0;849dee24a242:34123 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T21:26:56,172 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(959): stopping server 849dee24a242,34123,1732224229084 2024-11-21T21:26:56,172 INFO [RS:0;849dee24a242:34123 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:26:56,172 INFO [RS:0;849dee24a242:34123 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;849dee24a242:34123. 2024-11-21T21:26:56,172 DEBUG [RS:0;849dee24a242:34123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:26:56,172 DEBUG [RS:0;849dee24a242:34123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:56,173 INFO [RS:0;849dee24a242:34123 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T21:26:56,173 INFO [RS:0;849dee24a242:34123 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T21:26:56,173 INFO [RS:0;849dee24a242:34123 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T21:26:56,173 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T21:26:56,174 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T21:26:56,174 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T21:26:56,174 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:56,174 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:26:56,174 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:26:56,174 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:26:56,174 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:26:56,174 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:26:56,175 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T21:26:56,177 INFO [regionserver/849dee24a242:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:26:56,193 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/info/575f02b18a30456f9e379f717ae9fca2 is 95, key is TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1./info:/1732224264950/DeleteFamily/seqid=0 2024-11-21T21:26:56,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742514_1690 (size=6514) 2024-11-21T21:26:56,375 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:56,522 INFO [regionserver/849dee24a242:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T21:26:56,522 INFO [regionserver/849dee24a242:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T21:26:56,575 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:56,596 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/info/575f02b18a30456f9e379f717ae9fca2 2024-11-21T21:26:56,614 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/ns/3296c883d94a4ca5b3009147e87fe975 is 93, key is TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1./ns:/1732224264910/DeleteFamily/seqid=0 2024-11-21T21:26:56,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742515_1691 (size=5906) 2024-11-21T21:26:56,775 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:56,976 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:57,018 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/ns/3296c883d94a4ca5b3009147e87fe975 2024-11-21T21:26:57,045 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/rep_barrier/ee9c95df0fba4357a21549052894fc42 is 102, key is TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1./rep_barrier:/1732224264910/DeleteFamily/seqid=0 2024-11-21T21:26:57,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742516_1692 (size=6025) 2024-11-21T21:26:57,176 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:57,376 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:57,449 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/rep_barrier/ee9c95df0fba4357a21549052894fc42 2024-11-21T21:26:57,478 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/table/eceea152c6b2434d88b92002e389f748 is 96, key is TestAcidGuarantees,,1732224233903.4b5690ae4b378a5d396cbe15e83372d1./table:/1732224264910/DeleteFamily/seqid=0 2024-11-21T21:26:57,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742517_1693 (size=5936) 2024-11-21T21:26:57,576 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:57,777 DEBUG [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:26:57,883 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/table/eceea152c6b2434d88b92002e389f748 2024-11-21T21:26:57,893 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/info/575f02b18a30456f9e379f717ae9fca2 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/info/575f02b18a30456f9e379f717ae9fca2 2024-11-21T21:26:57,898 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/info/575f02b18a30456f9e379f717ae9fca2, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T21:26:57,899 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/ns/3296c883d94a4ca5b3009147e87fe975 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/ns/3296c883d94a4ca5b3009147e87fe975 2024-11-21T21:26:57,903 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/ns/3296c883d94a4ca5b3009147e87fe975, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T21:26:57,903 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/rep_barrier/ee9c95df0fba4357a21549052894fc42 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/rep_barrier/ee9c95df0fba4357a21549052894fc42 2024-11-21T21:26:57,908 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/rep_barrier/ee9c95df0fba4357a21549052894fc42, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T21:26:57,909 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/.tmp/table/eceea152c6b2434d88b92002e389f748 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/table/eceea152c6b2434d88b92002e389f748 2024-11-21T21:26:57,913 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/table/eceea152c6b2434d88b92002e389f748, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T21:26:57,914 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1739ms, sequenceid=90, compaction requested=false 2024-11-21T21:26:57,919 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T21:26:57,920 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:26:57,920 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:26:57,920 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732224416174Running coprocessor pre-close hooks at 1732224416174Disabling compacts and flushes for region at 1732224416174Disabling writes for close at 1732224416174Obtaining lock to block concurrent updates at 1732224416175 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732224416175Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732224416175Flushing stores of hbase:meta,,1.1588230740 at 1732224416175Flushing 1588230740/info: creating writer at 1732224416176 (+1 ms)Flushing 1588230740/info: appending metadata at 1732224416193 (+17 ms)Flushing 1588230740/info: closing flushed file at 1732224416193Flushing 1588230740/ns: creating writer at 1732224416600 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732224416614 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732224416614Flushing 1588230740/rep_barrier: creating writer at 1732224417028 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732224417045 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732224417045Flushing 1588230740/table: creating writer at 1732224417459 (+414 ms)Flushing 1588230740/table: appending metadata at 1732224417478 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732224417478Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d8fd3c8: reopening flushed file at 1732224417893 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79264754: reopening flushed file at 1732224417898 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101ebd62: reopening flushed file at 1732224417903 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67a70bd2: reopening flushed file at 1732224417908 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1739ms, sequenceid=90, compaction requested=false at 1732224417914 (+6 ms)Writing region close event to WAL at 1732224417916 (+2 ms)Running coprocessor post-close hooks at 1732224417920 (+4 ms)Closed at 1732224417920 2024-11-21T21:26:57,920 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T21:26:57,977 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(976): stopping server 849dee24a242,34123,1732224229084; all regions closed. 2024-11-21T21:26:57,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741834_1010 (size=25344) 2024-11-21T21:26:57,991 DEBUG [RS:0;849dee24a242:34123 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/oldWALs 2024-11-21T21:26:57,991 INFO [RS:0;849dee24a242:34123 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 849dee24a242%2C34123%2C1732224229084.meta:.meta(num 1732224233401) 2024-11-21T21:26:57,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741833_1009 (size=15902110) 2024-11-21T21:26:57,996 DEBUG [RS:0;849dee24a242:34123 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/oldWALs 2024-11-21T21:26:57,996 INFO [RS:0;849dee24a242:34123 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 849dee24a242%2C34123%2C1732224229084:(num 1732224232682) 2024-11-21T21:26:57,996 DEBUG [RS:0;849dee24a242:34123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:26:57,996 INFO [RS:0;849dee24a242:34123 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:26:57,996 INFO [RS:0;849dee24a242:34123 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:26:57,997 INFO [RS:0;849dee24a242:34123 {}] hbase.ChoreService(370): Chore service for: regionserver/849dee24a242:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T21:26:57,997 INFO [RS:0;849dee24a242:34123 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:26:57,997 INFO [regionserver/849dee24a242:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:26:57,998 INFO [RS:0;849dee24a242:34123 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34123 2024-11-21T21:26:58,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:26:58,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/849dee24a242,34123,1732224229084 2024-11-21T21:26:58,072 INFO [RS:0;849dee24a242:34123 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:26:58,074 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [849dee24a242,34123,1732224229084] 2024-11-21T21:26:58,093 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/849dee24a242,34123,1732224229084 already deleted, retry=false 2024-11-21T21:26:58,093 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 849dee24a242,34123,1732224229084 expired; onlineServers=0 2024-11-21T21:26:58,094 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '849dee24a242,35047,1732224227985' ***** 2024-11-21T21:26:58,094 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T21:26:58,094 INFO [M:0;849dee24a242:35047 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:26:58,094 INFO [M:0;849dee24a242:35047 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:26:58,095 DEBUG [M:0;849dee24a242:35047 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T21:26:58,095 DEBUG [M:0;849dee24a242:35047 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T21:26:58,095 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T21:26:58,095 DEBUG [master/849dee24a242:0:becomeActiveMaster-HFileCleaner.large.0-1732224232366 {}] cleaner.HFileCleaner(306): Exit Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.large.0-1732224232366,5,FailOnTimeoutGroup] 2024-11-21T21:26:58,095 DEBUG [master/849dee24a242:0:becomeActiveMaster-HFileCleaner.small.0-1732224232366 {}] cleaner.HFileCleaner(306): Exit Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.small.0-1732224232366,5,FailOnTimeoutGroup] 2024-11-21T21:26:58,096 INFO [M:0;849dee24a242:35047 {}] hbase.ChoreService(370): Chore service for: master/849dee24a242:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T21:26:58,096 INFO [M:0;849dee24a242:35047 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:26:58,096 DEBUG [M:0;849dee24a242:35047 {}] master.HMaster(1795): Stopping service threads 2024-11-21T21:26:58,096 INFO [M:0;849dee24a242:35047 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T21:26:58,097 INFO [M:0;849dee24a242:35047 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:26:58,097 ERROR [M:0;849dee24a242:35047 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T21:26:58,099 INFO [M:0;849dee24a242:35047 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T21:26:58,099 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T21:26:58,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T21:26:58,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:26:58,103 DEBUG [M:0;849dee24a242:35047 {}] zookeeper.ZKUtil(347): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T21:26:58,104 WARN [M:0;849dee24a242:35047 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T21:26:58,104 INFO [M:0;849dee24a242:35047 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/.lastflushedseqids 2024-11-21T21:26:58,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742518_1694 (size=72) 2024-11-21T21:26:58,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:26:58,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34123-0x1015f4813200001, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:26:58,184 INFO [RS:0;849dee24a242:34123 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:26:58,184 INFO [RS:0;849dee24a242:34123 {}] regionserver.HRegionServer(1031): Exiting; stopping=849dee24a242,34123,1732224229084; zookeeper connection closed. 2024-11-21T21:26:58,185 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e7f017e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e7f017e 2024-11-21T21:26:58,186 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T21:26:58,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:26:58,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T21:26:58,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:26:58,515 INFO [M:0;849dee24a242:35047 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T21:26:58,515 INFO [M:0;849dee24a242:35047 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T21:26:58,516 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:26:58,516 INFO [M:0;849dee24a242:35047 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:26:58,516 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:26:58,516 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:26:58,516 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:26:58,516 INFO [M:0;849dee24a242:35047 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=727.46 KB heapSize=892.20 KB 2024-11-21T21:26:58,538 DEBUG [M:0;849dee24a242:35047 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9792d6003f824c18b6a1143d7c908e15 is 82, key is hbase:meta,,1/info:regioninfo/1732224233480/Put/seqid=0 2024-11-21T21:26:58,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742519_1695 (size=5672) 2024-11-21T21:26:58,943 INFO [M:0;849dee24a242:35047 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2023 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9792d6003f824c18b6a1143d7c908e15 2024-11-21T21:26:58,970 DEBUG [M:0;849dee24a242:35047 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4477322775be4d7babc2f0c9840c2ab8 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1732224390126/Put/seqid=0 2024-11-21T21:26:58,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742520_1696 (size=44118) 2024-11-21T21:26:59,374 INFO [M:0;849dee24a242:35047 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=726.91 KB at sequenceid=2023 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4477322775be4d7babc2f0c9840c2ab8 2024-11-21T21:26:59,382 INFO [M:0;849dee24a242:35047 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4477322775be4d7babc2f0c9840c2ab8 2024-11-21T21:26:59,400 DEBUG [M:0;849dee24a242:35047 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f943e0ab43444b4c87100333a3c16656 is 69, key is 849dee24a242,34123,1732224229084/rs:state/1732224232424/Put/seqid=0 2024-11-21T21:26:59,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073742521_1697 (size=5156) 2024-11-21T21:26:59,804 INFO [M:0;849dee24a242:35047 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2023 (bloomFilter=true), to=hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f943e0ab43444b4c87100333a3c16656 2024-11-21T21:26:59,815 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9792d6003f824c18b6a1143d7c908e15 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9792d6003f824c18b6a1143d7c908e15 2024-11-21T21:26:59,819 INFO [M:0;849dee24a242:35047 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9792d6003f824c18b6a1143d7c908e15, entries=8, sequenceid=2023, filesize=5.5 K 2024-11-21T21:26:59,820 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4477322775be4d7babc2f0c9840c2ab8 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4477322775be4d7babc2f0c9840c2ab8 2024-11-21T21:26:59,823 INFO [M:0;849dee24a242:35047 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4477322775be4d7babc2f0c9840c2ab8 2024-11-21T21:26:59,824 INFO [M:0;849dee24a242:35047 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4477322775be4d7babc2f0c9840c2ab8, entries=170, sequenceid=2023, filesize=43.1 K 2024-11-21T21:26:59,824 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f943e0ab43444b4c87100333a3c16656 as hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f943e0ab43444b4c87100333a3c16656 2024-11-21T21:26:59,828 INFO [M:0;849dee24a242:35047 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33491/user/jenkins/test-data/d1e277c7-832b-3cfa-6c00-cba919a1820b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f943e0ab43444b4c87100333a3c16656, entries=1, sequenceid=2023, filesize=5.0 K 2024-11-21T21:26:59,829 INFO [M:0;849dee24a242:35047 {}] regionserver.HRegion(3140): Finished flush of dataSize ~727.46 KB/744921, heapSize ~891.91 KB/913312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2023, compaction requested=false 2024-11-21T21:26:59,830 INFO [M:0;849dee24a242:35047 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:26:59,831 DEBUG [M:0;849dee24a242:35047 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732224418516Disabling compacts and flushes for region at 1732224418516Disabling writes for close at 1732224418516Obtaining lock to block concurrent updates at 1732224418517 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732224418517Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=744921, getHeapSize=913552, getOffHeapSize=0, getCellsCount=2218 at 1732224418517Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732224418519 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732224418519Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732224418538 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732224418538Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732224418954 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732224418970 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732224418970Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732224419382 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732224419399 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732224419399Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18263cf4: reopening flushed file at 1732224419814 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19eaef38: reopening flushed file at 1732224419819 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2528f97a: reopening flushed file at 1732224419824 (+5 ms)Finished flush of dataSize ~727.46 KB/744921, heapSize ~891.91 KB/913312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2023, compaction requested=false at 1732224419829 (+5 ms)Writing region close event to WAL at 1732224419830 (+1 ms)Closed at 1732224419830 2024-11-21T21:26:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33455 is added to blk_1073741830_1006 (size=877615) 2024-11-21T21:26:59,834 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:26:59,834 INFO [M:0;849dee24a242:35047 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T21:26:59,834 INFO [M:0;849dee24a242:35047 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35047 2024-11-21T21:26:59,834 INFO [M:0;849dee24a242:35047 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:26:59,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:26:59,982 INFO [M:0;849dee24a242:35047 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:26:59,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35047-0x1015f4813200000, quorum=127.0.0.1:63066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:26:59,991 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T21:26:59,994 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T21:26:59,994 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T21:26:59,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T21:26:59,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/hadoop.log.dir/,STOPPED} 2024-11-21T21:26:59,998 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T21:26:59,998 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T21:26:59,999 WARN [BP-1334735833-172.17.0.2-1732224224328 heartbeating to localhost/127.0.0.1:33491 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T21:26:59,999 WARN [BP-1334735833-172.17.0.2-1732224224328 heartbeating to localhost/127.0.0.1:33491 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1334735833-172.17.0.2-1732224224328 (Datanode Uuid c7e979b6-f1e6-4ae2-a37e-979b86f3c671) service to localhost/127.0.0.1:33491 2024-11-21T21:27:00,002 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/data/data1/current/BP-1334735833-172.17.0.2-1732224224328 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T21:27:00,002 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/cluster_16549335-c81e-c2f7-a073-ad3dc05fe662/data/data2/current/BP-1334735833-172.17.0.2-1732224224328 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T21:27:00,003 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T21:27:00,009 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T21:27:00,010 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T21:27:00,010 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T21:27:00,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T21:27:00,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7493c1df-900a-cfd7-5b6a-1095438c200f/hadoop.log.dir/,STOPPED} 2024-11-21T21:27:00,025 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T21:27:00,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down